diff --git a/app/cli.py b/app/cli.py index 7ea0f1c..95878a0 100644 --- a/app/cli.py +++ b/app/cli.py @@ -152,8 +152,8 @@ def build_parser() -> argparse.ArgumentParser: prompt.add_argument("--remove", action="store_true", help="remove the prompt instead") stages = sub.add_parser( - "stages", help="show whether extract and summarise run, or switch one") - stages.add_argument("name", nargs="?", choices=["extract", "summarise"]) + "stages", help="show whether extract, summarise and enrich run, or switch one") + stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"]) stages.add_argument("state", nargs="?", choices=["on", "off"]) sub.add_parser("seniority", help="per host, how often the stated level equals the model's") diff --git a/app/jobs/stages.py b/app/jobs/stages.py index 7b89de9..968f0c1 100644 --- a/app/jobs/stages.py +++ b/app/jobs/stages.py @@ -13,8 +13,8 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`). asks no model and is on by default (`enrich.enabled`); its switch is `stage:enrich`, set the same way through `PUT /api/stages/enrich`. -`describe` is what `GET /api/stages` and `app stages` show: for each model -stage whether it is on, where that comes from, the model it uses, and who +`describe` is what `GET /api/stages` and `app stages` show: for each stage +with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who would answer. """ @@ -33,6 +33,8 @@ NAMES = ("extract", "summarise") #: Every stage with a switch: the model stages and `enrich`. SWITCHES = (*NAMES, "enrich") +#: What `describe_one` says of a stage that asks no model. +NO_ASK = "reads role pages, asks no model" NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, " "see docs/llm-backends.md") ANSWERS = { @@ -42,7 +44,7 @@ ANSWERS = { class Stage(BaseModel): - """One model stage as the page shows it. `source` is "stored" when the + """One stage as the page shows it. `source` is "stored" when the switch set it and "config" when it is still config.yaml's default. `backend` is `agent.backend`'s answer, "" for nobody, and `answer` says it in words.""" @@ -89,6 +91,10 @@ def answer(config: Config) -> tuple[str, str]: def describe_one(config: Config, store: Store, name: str) -> Stage: + if name not in NAMES: + return Stage(name=name, on=stage_on(config, store, name), + source="config" if stored(store, name) is None else "stored", + model="", backend="", answer=NO_ASK) backend, said = answer(config) return Stage(name=name, on=stage_on(config, store, name), source="config" if stored(store, name) is None else "stored", @@ -97,4 +103,4 @@ def describe_one(config: Config, store: Store, name: str) -> Stage: def describe(config: Config, store: Store) -> list[Stage]: - return [describe_one(config, store, name) for name in NAMES] + return [describe_one(config, store, name) for name in SWITCHES] diff --git a/app/static/jobs.js b/app/static/jobs.js index ea1a119..5e04de8 100644 --- a/app/static/jobs.js +++ b/app/static/jobs.js @@ -1,5 +1,5 @@ /* The Runs section of Sources: the "Run the pull now" button, and the - * switches for the two stages that ask a model. + * switches for the stages of a pull: the two that ask a model and enrich. * * The switches are `/api/stages`: one row per stage, drawn as the search * rows above are (`.srow`, `.switch`), and a switch is a PUT that stores the @@ -62,6 +62,7 @@ const DOES = { extract: "fills in country, seniority, salary and the like from what the posting says", summarise: "writes a short summary and tags for the best roles", + enrich: "reads the own page of the best roles whose list gave no text", }; const el = (tag, className, text) => { @@ -105,7 +106,7 @@ if (!stage.on) row.dataset.muted = "true"; const name = el("span", "name"); name.append(el("span", "", stage.name[0].toUpperCase() + stage.name.slice(1)), - el("span", "badge", stage.model)); + ...(stage.model ? [el("span", "badge", stage.model)] : [])); row.append(name, stageSwitch(stage, () => renderStages(stages))); const sub = el("span", "sub"); sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `); diff --git a/app/web/api.py b/app/web/api.py index 9c7b5a1..5d5a0c2 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -414,7 +414,7 @@ class StageBody(BaseModel): @router.get("/stages") def list_stages(store: StoreDep, config: ConfigDep) -> list[jobs.stages.Stage]: - """The two model stages of a pull: whether each is on, whether the + """The stages of a pull with a switch, extract, summarise and enrich: whether each is on, whether the stored switch or config.yaml says so, its model, and who would answer.""" return jobs.stages.describe(config, store) diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 9a4a2e2..c4b2806 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -178,6 +178,29 @@ posting per place, each with its own id, declares `places="copies"`: each copy is a row with its one place, and `dupes.py` folds them into one role that keeps each copy's link. +**Helpers.** `._shared` holds what more than one host needs; a module keeps +only its own word tables and uses these: + +- `place(text, remote=None)` makes a `Place`: remote when the text has the + word `remote`, otherwise as `remote` says. `add_place(found, entry)` + appends a place unless its text is empty or already in `found`, in the + host's order. A host that states remote itself passes it to `place`, or + builds the dict and passes that to `add_place`. +- `table_word(table, value, log, what)` maps a host's word (stated seniority + or employment) through the module's table, without case. The table's keys + are lowercase. Empty gives None; a known word mapped to `""` (`experienced`, + `permanent`) names no level and gives None; an unknown word gives None and + a debug line naming the host, so the table can grow. +- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`, + `contract`, `temporary`, `internship`, `freelance`. Every value of a + module's `_EMPLOYMENT` table is one of these or `""`; the test in + `tests/test_readers_shared.py` holds every module to it. Seniority values + are `app.models.SENIORITIES`. +- `country_if_single(codes)` is the code when the role's places share exactly + one, else None, for a host that states a country per place. +- `remote_word(text)` reads `remote` or `hybrid` from a search provider's + headline or town line (Danish, Swedish, English). + **Teaser.** A short text the list carries (an excerpt, a summary) is handed over as `teaser`, plain text, whether or not the full text comes later. It is shown and embedded until `description` arrives. A reader that sees one diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index f76accc..6894556 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -8,6 +8,7 @@ import it without either importing the other. The leading underscore keeps from __future__ import annotations +import logging import re from collections.abc import Callable from datetime import UTC, datetime @@ -16,6 +17,8 @@ from typing import Any, Protocol import httpx +from ._contract import Place + #: What an undeclared module's parser turns a feed into; a module that #: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`. #: One dict per row, with the keys @@ -192,3 +195,59 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> tail = f" per {period}" if period else "" code = str(currency or "").strip().upper() return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" + + +#: The employment words a row's `employment` may hold, for every host. A +#: host's table maps its own words onto these; a word that names none of them +#: (a permanent contract says nothing about hours) maps to "". +EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance") + +_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE) + + +def place(text: str, remote: bool | None = None) -> Place: + """A `Place` for `text`: remote when the text says the word `remote`, + otherwise as `remote` says (None when the host says nothing).""" + return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote} + + +def add_place(found: list[Place], entry: Place) -> None: + """Append `entry` to `found` unless its text is empty or already there; + the host's order is kept.""" + if entry["text"] and entry["text"] not in (p["text"] for p in found): + found.append(entry) + + +def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None: + """`table`'s word for a host's `value`, looked up without case. None when + the host states none, when the table maps the word to "" (a known word + that names no level, such as `experienced`), or when the table does not + know it; that last case is logged at debug level so the table can grow.""" + word = str(value or "").strip() + if not word: + return None + found = table.get(word.lower()) + if found is None: + log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word) + return found or None + + +def country_if_single(codes: Any) -> str | None: + """The one code among `codes` when there is exactly one, else None: a + country is stated for a role only when all its places share it.""" + return next(iter(codes)) if len(codes) == 1 else None + + +#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". +_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) +_REMOTE = re.compile( + r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) + + +def remote_word(text: str) -> str | None: + """`remote`, `hybrid` or None: the word a search provider's headline or + town line says about where the work is done, in Danish, Swedish and + English. A board states its own remote flag and needs none of this.""" + if _REMOTE.search(text): + return "remote" + return "hybrid" if _HYBRID.search(text) else None diff --git a/plugins/readers/_words.py b/plugins/readers/_words.py deleted file mode 100644 index e8d4cba..0000000 --- a/plugins/readers/_words.py +++ /dev/null @@ -1,19 +0,0 @@ -"""What the search providers' headlines and town lines say about where the -work is done, in Danish, Swedish and English. Shared by the search readers; -a board states its own remote flag and needs none of this.""" - -from __future__ import annotations - -import re - -#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". -_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) -_REMOTE = re.compile( - r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE) - - -def remote_word(text: str) -> str | None: - """`remote`, `hybrid` or None: the word the text says.""" - if _REMOTE.search(text): - return "remote" - return "hybrid" if _HYBRID.search(text) else None diff --git a/plugins/readers/adzuna.py b/plugins/readers/adzuna.py index e5957c0..66067b0 100644 --- a/plugins/readers/adzuna.py +++ b/plugins/readers/adzuna.py @@ -24,8 +24,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Key, Place, Reads -from ._words import remote_word -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word NAME = "adzuna" LABEL = "Adzuna" diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py index 0e7b12c..dd70ae6 100644 --- a/plugins/readers/ashby.py +++ b/plugins/readers/ashby.py @@ -9,14 +9,13 @@ from __future__ import annotations import json import logging -import re from collections.abc import Callable from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount +from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount log = logging.getLogger(__name__) @@ -42,7 +41,7 @@ _API = "api.ashbyhq.com" _WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"} #: `employmentType` -> the app's employment words. -_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern", +_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship", "contract": "contract", "temporary": "temporary"} @@ -124,20 +123,12 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]: for s in j.get("secondaryLocations") or []] out: list[Place] = [] for text, remote in named: - if text and text not in (p["text"] for p in out): - out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) - else remote}) + add_place(out, place(text, remote)) return out def _employment(value: Any) -> str | None: - word = str(value or "").strip() - if not word: - return None - found = _EMPLOYMENT.get(word.lower()) - if found is None: - log.debug("ashby: unknown employmentType %r", word) - return found + return table_word(_EMPLOYMENT, value, log, "employmentType") def parse(body: bytes, slug: str) -> list[Row]: diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py index ab3fa3a..95de9e9 100644 --- a/plugins/readers/bamboohr.py +++ b/plugins/readers/bamboohr.py @@ -17,14 +17,13 @@ from __future__ import annotations import html import json import logging -import re from collections.abc import Callable from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, label_slug_ok, only, parse_dt +from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word log = logging.getLogger(__name__) @@ -39,7 +38,7 @@ READS = Reads( #: the company's own ("Full-Time", "Part-Time", "Contract", "Intern"). _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", "part time": "part_time", "contract": "contract", "contractor": "contract", - "temporary": "temporary", "intern": "intern", "internship": "intern", + "temporary": "temporary", "intern": "internship", "internship": "internship", "freelance": "freelance"} NAME = "bamboohr" @@ -97,18 +96,11 @@ def _place(j: dict[str, Any]) -> str: def _places(j: dict[str, Any]) -> list[Place]: """The one place BambooHR states, if it states any.""" text = _place(j) - return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \ - if text else [] + return [place(text)] if text else [] def _employment(j: dict[str, Any]) -> str | None: - label = str(j.get("employmentStatusLabel") or "").strip() - if not label: - return None - found = _EMPLOYMENT.get(label.lower()) - if found is None: - log.debug("bamboohr: unknown employmentStatusLabel %r", label) - return found + return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel") def parse(body: bytes, slug: str) -> list[Row]: diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py index 521276e..54b51b6 100644 --- a/plugins/readers/deel.py +++ b/plugins/readers/deel.py @@ -24,7 +24,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import num, only, parse_dt, pay_line +from ._shared import num, only, parse_dt, pay_line, table_word log = logging.getLogger(__name__) @@ -65,8 +65,8 @@ _REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} #: The name of a `jobEmploymentTypes` entry -> the app's employment words. _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", "part time": "part_time", "contract": "contract", "contractor": "contract", - "consultant": "contract", "temporary": "temporary", "intern": "intern", - "internship": "intern", "freelance": "freelance"} + "consultant": "contract", "temporary": "temporary", "intern": "internship", + "internship": "internship", "freelance": "freelance"} def slug_ok(slug: str) -> bool: @@ -123,12 +123,9 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]: def _employment(job: dict[str, Any]) -> str | None: """The first `jobEmploymentTypes` entry that names a known type.""" for entry in job.get("jobEmploymentTypes") or []: - name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip() - if not name: - continue - if found := _EMPLOYMENT.get(name.lower()): + name = ((entry or {}).get("employmentType") or {}).get("name") + if found := table_word(_EMPLOYMENT, name, log, "employment type"): return found - log.debug("deel: unknown employment type %r", name) return None diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index 075a7c9..e015f82 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -44,7 +44,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads, Row from ._countries import name_of -from ._shared import Get, Page, is_web_url, only, parse_dt +from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place READS = Reads( scope="board", listing="paged", page_size=10, text="detail", complete="stated", @@ -131,8 +131,7 @@ def _places(locations: Any) -> list[Place]: if name and name not in names: names.append(name) text = ", ".join(names) - if text and text not in (p["text"] for p in out): - out.append({"text": text, "remote": True if "remote" in text.lower() else None}) + add_place(out, place(text)) return out diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py index 1d225df..bb8c6d7 100644 --- a/plugins/readers/gem.py +++ b/plugins/readers/gem.py @@ -27,7 +27,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, only, parse_dt +from ._shared import Get, Page, add_place, only, parse_dt, table_word log = logging.getLogger(__name__) @@ -69,9 +69,9 @@ DETAIL_QUERY = ( _REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} #: `job.employmentType` -> the app's employment words. -_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract", - "CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern", - "TEMPORARY": "temporary", "FREELANCE": "freelance"} +_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract", + "contractor": "contract", "intern": "internship", "internship": "internship", + "temporary": "temporary", "freelance": "freelance"} def slug_ok(slug: str) -> bool: @@ -131,8 +131,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: places: list[Place] = [] for loc in locations: text = str(loc.get("city") or loc.get("name") or "").strip() - if text and text not in (pl["text"] for pl in places): - places.append({"text": text, "remote": loc["isRemote"] + add_place(places, {"text": text, "remote": loc["isRemote"] if isinstance(loc.get("isRemote"), bool) else None}) job = p.get("job") or {} remote = _REMOTE.get(str(job.get("locationType") or "").upper()) @@ -150,11 +149,8 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: "remote": remote, "salary_text": "", } - if word := str(job.get("employmentType") or "").strip().upper(): - if employment := _EMPLOYMENT.get(word): - row["employment"] = employment - else: - log.debug("gem: unknown employmentType %r", word) + if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"): + row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py index 3753026..c79d31d 100644 --- a/plugins/readers/greenhouse.py +++ b/plugins/readers/greenhouse.py @@ -15,7 +15,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line +from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place NAME = "greenhouse" LABEL = "Greenhouse" @@ -132,19 +132,16 @@ def _places(j: dict[str, Any]) -> list[Place]: out: list[Place] = [] for office in j.get("offices") or []: text = str((office or {}).get("location") or (office or {}).get("name") or "").strip() - if text and text not in (p["text"] for p in out): - out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) - else None}) + add_place(out, place(text)) if not out or not name or _PLACEHOLDER.fullmatch(name): - return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) - else None}] if name else []) + return out or ([place(name)] if name else []) # A name that says something the offices do not (a "Remote India" beside # an office in Bangalore) is one more place; one they already contain, # or that contains them, is the same place said twice. low = name.lower() if any(p["text"].lower() in low or low in p["text"].lower() for p in out): return out - return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out] + return [place(name), *out] def parse(body: bytes, slug: str) -> list[Row]: diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py index fa5349a..31fa9df 100644 --- a/plugins/readers/hibob.py +++ b/plugins/readers/hibob.py @@ -23,7 +23,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount log = logging.getLogger(__name__) @@ -44,7 +44,7 @@ _EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time", "full-time": "full_time", "regular part time": "part_time", "part time": "part_time", "part-time": "part_time", "contract": "contract", "contractor": "contract", "temporary": "temporary", "fixed term": "temporary", - "intern": "intern", "internship": "intern", "freelance": "freelance"} + "intern": "internship", "internship": "internship", "freelance": "freelance"} NAME = "hibob" LABEL = "HiBob" @@ -132,11 +132,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "salary_max": salary_max, "currency": code, } - kind = str(ad.get("employmentType") or "").strip() - if kind: - if employment := _EMPLOYMENT.get(kind.lower()): - row["employment"] = employment - else: - log.debug("hibob: unknown employmentType %r", kind) + if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"): + row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py index 9314675..0e12946 100644 --- a/plugins/readers/jibe.py +++ b/plugins/readers/jibe.py @@ -38,7 +38,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, only, parse_dt +from ._shared import Get, Page, add_place, only, parse_dt, place, table_word log = logging.getLogger(__name__) @@ -50,9 +50,9 @@ READS = Reads( ) #: `employment_type` -> the app's employment words. -_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract", - "CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern", - "INTERNSHIP": "intern", "FREELANCE": "freelance"} +_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract", + "contract": "contract", "temporary": "temporary", "intern": "internship", + "internship": "internship", "freelance": "freelance"} NAME = "jibe" LABEL = "Jibe" @@ -120,9 +120,7 @@ def _places(j: dict[str, Any]) -> list[Place]: out: list[Place] = [] for part in line.split(";"): text = part.strip() - if text and text not in (p["text"] for p in out): - out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) - else None}) + add_place(out, place(text)) return out @@ -162,11 +160,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: code = str(j.get("country_code") or "").strip().upper() if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code): row["country"] = code - kind = str(j.get("employment_type") or "").strip() - if kind: - if employment := _EMPLOYMENT.get(kind.upper()): - row["employment"] = employment - else: - log.debug("jibe: unknown employment_type %r", kind) + if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"): + row["employment"] = employment out.append(row) return out diff --git a/plugins/readers/jobindex.py b/plugins/readers/jobindex.py index 012acaa..0dd6da9 100644 --- a/plugins/readers/jobindex.py +++ b/plugins/readers/jobindex.py @@ -28,8 +28,7 @@ from urllib.parse import urlencode from desk.sources import strip_html -from ._words import remote_word -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word from ._contract import Ask, Fields, Filters, Place, Reads NAME = "jobindex" diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py index a631107..74fc7b3 100644 --- a/plugins/readers/jobylon.py +++ b/plugins/readers/jobylon.py @@ -25,8 +25,8 @@ from typing import Any from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, parse_dt +from ._contract import Ask, Fields, Reads, Row +from ._shared import Get, parse_dt, place, table_word log = logging.getLogger(__name__) @@ -70,7 +70,7 @@ _SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern", #: The embed's `employment_type` -> the app's employment words. _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time", "part time": "part_time", "contract": "contract", "consultant": "contract", - "temporary": "temporary", "internship": "intern", "intern": "intern", + "temporary": "temporary", "internship": "internship", "intern": "internship", "freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"} _ASSIGNMENT = "JBL.embed_v2['jobs'] = " @@ -179,22 +179,6 @@ def _published(value: Any) -> Any: return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}") -def _mapped(table: dict[str, str], value: Any, what: str) -> str | None: - """`table`'s word for the host's `value`; None when the host states none - or the table does not know it (logged, so the table can grow).""" - word = str(value or "").strip() - if not word: - return None - found = table.get(word.lower()) - if found is None: - log.debug("jobylon: unknown %s %r", what, word) - return found or None - - -def _place(text: str) -> Place: - return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None} - - def parse(body: bytes, slug: str) -> list[Row]: """The embed page -> one row per role. `posted` is the day `published_date` names, which `detail` replaces with the exact time. `location` joins the @@ -210,7 +194,7 @@ def parse(body: bytes, slug: str) -> list[Row]: if not title or not job_id or not _ROLE_PATH.fullmatch(path): continue function = str(j.get("function") or "").strip() - places = [_place(name) for name in dict.fromkeys( + places = [place(name) for name in dict.fromkeys( str(name).strip() for name in j.get("locations") or [] if str(name).strip())] row: Row = { "source_id": job_id, @@ -229,9 +213,9 @@ def parse(body: bytes, slug: str) -> list[Row]: summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split()) if summary and summary != "None": row["teaser"] = summary - if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"): + if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"): row["seniority"] = seniority - if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"): + if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"): row["employment"] = employment out.append(row) return out @@ -252,8 +236,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: else: raise ValueError("no JobPosting on the role's page") places = [] - for place in data.get("jobLocation") or []: - address = (place or {}).get("address") or {} + for loc in data.get("jobLocation") or []: + address = (loc or {}).get("address") or {} name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip() if name and name not in places: places.append(name) @@ -263,5 +247,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: } if places: found["location"] = "; ".join(places) - found["places"] = [_place(name) for name in places] + found["places"] = [place(name) for name in places] return found diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index f2bca21..f953792 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -24,7 +24,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount +from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ READS = Reads( #: text on Lever ("Permanent" is seen too and says nothing about hours). _EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time", "part-time": "part_time", "part time": "part_time", "parttime": "part_time", - "intern": "intern", "internship": "intern", "contract": "contract", + "intern": "internship", "internship": "internship", "contract": "contract", "contractor": "contract", "temporary": "temporary", "freelance": "freelance"} NAME = "lever" @@ -115,9 +115,7 @@ def _places(categories: dict[str, Any]) -> list[Place]: out: list[Place] = [] for name in names: text = str(name or "").strip() - if text and text not in (p["text"] for p in out): - out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) - else None}) + add_place(out, place(text)) return out @@ -158,11 +156,7 @@ def parse(body: bytes, slug: str) -> list[Row]: country = str(j.get("country") or "").strip().upper() if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country): row["country"] = country - commitment = str(categories.get("commitment") or "").strip() - if commitment: - if employment := _EMPLOYMENT.get(commitment.lower()): - row["employment"] = employment - else: - log.debug("lever: unknown commitment %r", commitment) + if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"): + row["employment"] = employment rows.append(row) return rows diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index c02826f..9ac97eb 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -35,7 +35,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES, code_of -from ._shared import Get, Page, Row, only, parse_dt +from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt NAME = "oracle" LABEL = "Oracle Recruiting Cloud" @@ -197,8 +197,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: _text(role.get("PrimaryLocationCountry"))), *((_text(s.get("Name")), _text(s.get("CountryCode"))) for s in secondary)): - if name and name not in (p["text"] for p in places): - places.append({"text": name, "remote": {"remote": True, "hybrid": False, + add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False, "onsite": False}.get(remote or "")}) if code: codes.add(code.upper()) @@ -213,7 +212,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: or _text(role.get("Department")) or _text(role.get("Organization"))), "description": "", "teaser": _plain(role.get("ShortDescriptionStr")), - "country": next(iter(codes)) if len(codes) == 1 else None, + "country": country_if_single(codes), "remote": remote, "salary_text": "", # the resource states no pay }) diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py index 8427b89..6476768 100644 --- a/plugins/readers/personio.py +++ b/plugins/readers/personio.py @@ -14,7 +14,7 @@ from collections.abc import Callable from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount log = logging.getLogger(__name__) @@ -86,12 +86,7 @@ def _salary_text(salary_info: ET.Element) -> str: def _seniority(word: str) -> str | None: - key = word.strip().lower() - if key not in _SENIORITY: - if key: - log.debug("personio: unknown seniority %r", word) - return None - return _SENIORITY[key] or None + return table_word(_SENIORITY, word, log, "seniority") def _employment(position: ET.Element) -> str | None: @@ -113,8 +108,7 @@ def _places(position: ET.Element) -> list[Place]: *(o.text for o in position.findall("additionalOffices/office"))] for name in names: text = (name or "").strip() - if text and text not in (p["text"] for p in found): - found.append({"text": text, "remote": True if "remote" in text.lower() else None}) + add_place(found, place(text)) return found diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index caa623c..1c62876 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -31,7 +31,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import name_of -from ._shared import Get, Page, Row, only, parse_dt +from ._shared import Get, Page, Row, add_place, only, parse_dt NAME = "phenom" LABEL = "Phenom" @@ -163,10 +163,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: if not title or not key: continue places: list[Place] = [] - for place in (job.get("location"), *(job.get("multi_location") or [])): - if isinstance(place, str) and place.strip() \ - and place.strip() not in (p["text"] for p in places): - places.append({"text": place.strip(), "remote": None}) + for text in (job.get("location"), *(job.get("multi_location") or [])): + if isinstance(text, str): + add_place(places, {"text": text.strip(), "remote": None}) tail = _slug_of(title) out.append({ "source_id": key, diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py index 6f2e246..48fd6ff 100644 --- a/plugins/readers/platsbanken.py +++ b/plugins/readers/platsbanken.py @@ -22,8 +22,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt -from ._words import remote_word +from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word NAME = "platsbanken" LABEL = "Platsbanken" diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py index 127158c..e494d4c 100644 --- a/plugins/readers/recruitee.py +++ b/plugins/readers/recruitee.py @@ -15,7 +15,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount log = logging.getLogger(__name__) @@ -83,12 +83,7 @@ def _salary_text(salary: dict[str, Any]) -> str: def _seniority(code: str) -> str | None: - key = code.strip().lower() - if key not in _SENIORITY: - if key: - log.debug("recruitee: unknown experience_code %r", code) - return None - return _SENIORITY[key] or None + return table_word(_SENIORITY, code, log, "experience_code") def _employment(code: str) -> str | None: @@ -112,8 +107,8 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]: def add(text: str) -> None: text = text.strip() - if text and text.lower() not in _NO_PLACE and text not in (p["text"] for p in found): - found.append({"text": text, "remote": None}) + if text.lower() not in _NO_PLACE: + add_place(found, {"text": text, "remote": None}) for loc in offer.get("locations") or []: if not isinstance(loc, dict): @@ -165,7 +160,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "salary_min": salary_min, "salary_max": salary_max, "currency": currency, - "country": next(iter(codes)) if len(codes) == 1 else None, + "country": country_if_single(codes), "seniority": _seniority(str(j.get("experience_code") or "")), "employment": _employment(str(j.get("employment_type_code") or "")), }) diff --git a/plugins/readers/rippling.py b/plugins/readers/rippling.py index 171d4e8..866e603 100644 --- a/plugins/readers/rippling.py +++ b/plugins/readers/rippling.py @@ -20,7 +20,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Row, parse_dt, yearly_amount +from ._shared import Get, Row, add_place, parse_dt, yearly_amount log = logging.getLogger(__name__) @@ -100,8 +100,8 @@ def parse(body: bytes, slug: str) -> list[Row]: continue place = str(((entry.get("workLocation") or {}).get("label")) or "").strip() places.setdefault(job_id, []) - if place and place not in (p["text"] for p in places[job_id]): - places[job_id].append(_place(place)) + if place: + add_place(places[job_id], _place(place)) roles.setdefault(job_id, { "source_id": job_id, "title": title, diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index b58907f..152bb7f 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -19,7 +19,7 @@ from collections.abc import Callable from typing import Any from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok +from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word log = logging.getLogger(__name__) @@ -97,10 +97,7 @@ def parse(body: bytes, slug: str) -> list[Row]: def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None: """The `id` of the object `job[key]` looked up in `table`; none for a word the table maps to none, and a word it does not know is logged.""" - word = str((job.get(key) or {}).get("id") or "").strip().lower() - if word and word not in table: - log.debug("smartrecruiters: unknown %s %r", key, word) - return table.get(word) or None + return table_word(table, (job.get(key) or {}).get("id"), log, key) def rows(data: dict[str, Any], slug: str) -> list[Row]: diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 0124020..96d5858 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -59,7 +59,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt NAME = "successfactors" LABEL = "SuccessFactors" @@ -392,13 +392,12 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: addresses = _addresses(text) or [meta] places: list[Place] = [] for address in addresses: - if (place := _location(address)) and place not in (p["text"] for p in places): - places.append({"text": place, "remote": None}) + add_place(places, {"text": _location(address), "remote": None}) if places: out["places"] = places out["location"] = "; ".join(p["text"] for p in places) codes = {_country_code(address) for address in addresses} - if len(codes) == 1 and (code := next(iter(codes))): + if code := country_if_single(codes): out["country"] = code return out diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py index f9f7366..075e7e5 100644 --- a/plugins/readers/teamtailor.py +++ b/plugins/readers/teamtailor.py @@ -17,7 +17,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt NAME = "teamtailor" LABEL = "Teamtailor" @@ -129,9 +129,8 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: for loc in item.findall(f"{_TT}locations/{_TT}location"): city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip() country = (loc.findtext(f"{_TT}country") or "").strip() - place = city if country == city else ", ".join(p for p in (city, country) if p) - if place and place not in (p["text"] for p in places): - places.append({"text": place, "remote": remote}) + text = city if country == city else ", ".join(p for p in (city, country) if p) + add_place(places, {"text": text, "remote": remote}) out.append({ "source_id": (item.findtext("guid") or url).strip(), "title": title, diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py index 3a6cc52..99597ed 100644 --- a/plugins/readers/workable.py +++ b/plugins/readers/workable.py @@ -13,7 +13,7 @@ from collections.abc import Callable from typing import Any from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok +from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word log = logging.getLogger(__name__) @@ -82,8 +82,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]: if part and part not in parts: parts.append(part) text = ", ".join(parts) - if text and text not in (p["text"] for p in found): - found.append({"text": text, "remote": remote}) + add_place(found, {"text": text, "remote": remote}) if code := str(loc.get("countryCode") or "").strip().upper(): codes.add(code) return found, codes @@ -107,9 +106,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: continue remote = bool(j.get("remote")) or j.get("workplace") == "remote" places, codes = _places(j, remote) - kind = str(j.get("type") or "").strip().lower() - if kind and kind not in _EMPLOYMENT: - log.debug("workable: unknown type %r", kind) dept = j.get("department") or "" if isinstance(dept, list): dept = ", ".join(d for d in dept if d) @@ -124,7 +120,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "description": "", "remote": remote, "salary_text": "", # Workable's list endpoint states no pay at all - "country": next(iter(codes)) if len(codes) == 1 else None, - "employment": _EMPLOYMENT.get(kind), + "country": country_if_single(codes), + "employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"), }) return out diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index fb9b25e..a2e6ec6 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -39,7 +39,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import code_of -from ._shared import Get, Page, Row, only, parse_dt +from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word log = logging.getLogger(__name__) @@ -320,7 +320,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: out["location"] = "; ".join(p["text"] for p in places) if len(places) == 1 and (code := _country_code(posting)): out["country"] = code - if employment := _EMPLOYMENT.get(str(posting.get("timeType") or "").strip().lower()): + if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"): out["employment"] = employment if posted := parse_dt(posting.get("startDate")): out["posted"] = posted @@ -344,9 +344,8 @@ def _places(posting: dict[str, Any], remote: str | None) -> list[Place]: (LEGO's "Billund") gets it from the role's `country`.""" found: list[Place] = [] for name in [posting.get("location"), *(posting.get("additionalLocations") or [])]: - place = _place(name, remote) - if place and place["text"] not in (p["text"] for p in found): - found.append(place) + if place := _place(name, remote): + add_place(found, place) country = str((posting.get("country") or {}).get("descriptor") or "").strip() if len(found) == 1 and country and country.lower() not in found[0]["text"].lower(): found[0]["text"] += f", {country}" diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py index 6038de1..90780a3 100644 --- a/tests/test_ats_jobylon.py +++ b/tests/test_ats_jobylon.py @@ -141,7 +141,7 @@ def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged( assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values()) rows = {r["source_id"]: r for r in jobylon.parse( (FIXTURES / "jobylon-places.html").read_bytes(), "3143")} - assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "intern" + assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship" def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None: diff --git a/tests/test_readers_jobindex.py b/tests/test_readers_jobindex.py index d45caa1..4cd9099 100644 --- a/tests/test_readers_jobindex.py +++ b/tests/test_readers_jobindex.py @@ -56,7 +56,7 @@ def test_it_reads_title_company_town_date_and_teaser() -> None: def test_it_reads_hybrid_and_remote_from_the_town_line() -> None: - from plugins.readers._words import remote_word + from plugins.readers._shared import remote_word assert remote_word("Birkerød, Hybrid position") == "hybrid" assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid" diff --git a/tests/test_readers_shared.py b/tests/test_readers_shared.py new file mode 100644 index 0000000..91b3474 --- /dev/null +++ b/tests/test_readers_shared.py @@ -0,0 +1,85 @@ +"""The helpers the host modules share, and the one employment vocabulary.""" + +from __future__ import annotations + +import logging +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins import readers # noqa: E402 +from plugins.readers import _shared # noqa: E402 +from plugins.readers._shared import ( # noqa: E402 + EMPLOYMENTS, + add_place, + country_if_single, + place, + remote_word, + table_word, +) + +#: The module-level names a reader keeps its employment words in. +_TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE") + + +def test_every_employment_table_maps_into_the_shared_vocabulary() -> None: + found = 0 + for name, module in readers.READERS.items(): + for table_name in _TABLES: + table = getattr(module, table_name, None) + if table is None: + continue + found += 1 + values = table.values() if isinstance(table, dict) else [v for _k, v in table] + # "" is a known word that names no employment ("permanent"). + assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name) + assert found >= 15 + + +def test_the_vocabulary_is_the_documented_one() -> None: + assert EMPLOYMENTS == ("full_time", "part_time", "contract", "temporary", "internship", + "freelance") + + +def test_place_is_remote_on_the_word_and_otherwise_as_stated() -> None: + assert place("Remote (DE)") == {"text": "Remote (DE)", "remote": True} + assert place("Berlin, remote") == {"text": "Berlin, remote", "remote": True} + assert place("Berlin") == {"text": "Berlin", "remote": None} + assert place("Berlin", False) == {"text": "Berlin", "remote": False} + assert place("Remote", False)["remote"] is True + assert place("Remotely", None)["remote"] is None + + +def test_add_place_skips_empty_and_repeated_texts_and_keeps_order() -> None: + found: list = [] + for text in ("Berlin", "", "Oslo", "Berlin"): + add_place(found, place(text)) + assert [p["text"] for p in found] == ["Berlin", "Oslo"] + + +def test_table_word_maps_ignores_case_and_logs_only_unknown_words(caplog) -> None: + table = {"full-time": "full_time", "permanent": ""} + log = logging.getLogger("plugins.readers.acme") + with caplog.at_level(logging.DEBUG, logger=log.name): + assert table_word(table, " Full-Time ", log, "type") == "full_time" + assert table_word(table, "Permanent", log, "type") is None + assert table_word(table, None, log, "type") is None + assert not caplog.records + assert table_word(table, "Odd", log, "type") is None + assert [r.getMessage() for r in caplog.records] == ["acme: unknown type 'Odd'"] + + +def test_country_if_single_needs_exactly_one_code() -> None: + assert country_if_single({"DE"}) == "DE" + assert country_if_single({"DE", "SE"}) is None + assert country_if_single(set()) is None + + +def test_remote_word_reads_danish_swedish_and_english() -> None: + assert remote_word("Remote, Danmark") == "remote" + assert remote_word("Birkerød, Hybrid position") == "hybrid" + assert remote_word("Hedehusene") is None + assert _shared.remote_word is remote_word diff --git a/tests/test_stages.py b/tests/test_stages.py index 08bf321..3955384 100644 --- a/tests/test_stages.py +++ b/tests/test_stages.py @@ -143,7 +143,7 @@ def client(app: FastAPI) -> Iterator[TestClient]: yield c -def test_get_stages_lists_both_with_their_source_model_and_who_answers( +def test_get_stages_lists_all_three_with_their_source_model_and_who_answers( client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: answered_by(monkeypatch, "api") res = client.get("/api/stages") @@ -156,6 +156,8 @@ def test_get_stages_lists_both_with_their_source_model_and_who_answers( {"name": "summarise", "on": False, "source": "config", "model": "claude-haiku-4-5-20251001", "backend": "api", "answer": "answered by the Claude API"}, + {"name": "enrich", "on": True, "source": "config", "model": "", "backend": "", + "answer": "reads role pages, asks no model"}, ] @@ -164,7 +166,7 @@ def test_who_answers_is_said_for_the_api_claude_code_and_nobody( answers = {} for backend in ("api", "claude_code", ""): answered_by(monkeypatch, backend) - [first, second] = client.get("/api/stages").json() + [first, second, _enrich] = client.get("/api/stages").json() assert first["backend"] == second["backend"] == backend answers[backend] = first["answer"] @@ -191,6 +193,18 @@ def test_put_stores_the_setting_returns_the_same_shape_and_starts_nothing( assert res.json()["on"] is False and store.state["stage:summarise"] == "off" +def test_the_enrich_switch_is_listed_and_stored_like_the_others( + client: TestClient, store: Store) -> None: + res = client.put("/api/stages/enrich", json={"on": False}) + + assert res.status_code == 200 + assert res.json()["name"] == "enrich" and res.json()["on"] is False + assert store.state["stage:enrich"] == "off" + listed = {stage["name"]: stage for stage in client.get("/api/stages").json()} + assert list(listed) == ["extract", "summarise", "enrich"] + assert listed["enrich"]["on"] is False and listed["enrich"]["source"] == "stored" + + def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None: assert client.put("/api/stages/embed", json={"on": True}).status_code == 404 assert client.put("/api/stages/extract", json={"on": "sometimes"}).status_code == 422 @@ -200,7 +214,7 @@ def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> No def test_agent_backend_off_is_named_rather_than_blamed_on_missing_keys( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: Store) -> None: off = Config(database=str(tmp_path / "roledesk.db"), agent={"backend": "off"}) - [first, _] = stages.describe(off, store) + [first, *_] = stages.describe(off, store) assert first.backend == "" and "agent.backend" in first.answer