Share the readers' place, table-lookup and country helpers, unify the employment vocabulary, and list the enrich switch with the stages

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 10:13:24 +02:00
1 parent ca80219bbb
commit 903b69b6d3
34 files changed
+290 -207

No files matched your search

+2 -2
View File
@@ -152,8 +152,8 @@ def build_parser() -> argparse.ArgumentParser:
prompt.add_argument("--remove", action="store_true", help="remove the prompt instead")
stages = sub.add_parser(
"stages", help="show whether extract and summarise run, or switch one")
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
"stages", help="show whether extract, summarise and enrich run, or switch one")
stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"])
stages.add_argument("state", nargs="?", choices=["on", "off"])
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
+10 -4
View File
@@ -13,8 +13,8 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`describe` is what `GET /api/stages` and `app stages` show: for each model
stage whether it is on, where that comes from, the model it uses, and who
`describe` is what `GET /api/stages` and `app stages` show: for each stage
with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
would answer.
"""
@@ -33,6 +33,8 @@ NAMES = ("extract", "summarise")
#: Every stage with a switch: the model stages and `enrich`.
SWITCHES = (*NAMES, "enrich")
#: What `describe_one` says of a stage that asks no model.
NO_ASK = "reads role pages, asks no model"
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
"see docs/llm-backends.md")
ANSWERS = {
@@ -42,7 +44,7 @@ ANSWERS = {
class Stage(BaseModel):
"""One model stage as the page shows it. `source` is "stored" when the
"""One stage as the page shows it. `source` is "stored" when the
switch set it and "config" when it is still config.yaml's default.
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
it in words."""
@@ -89,6 +91,10 @@ def answer(config: Config) -> tuple[str, str]:
def describe_one(config: Config, store: Store, name: str) -> Stage:
if name not in NAMES:
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model="", backend="", answer=NO_ASK)
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
@@ -97,4 +103,4 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
def describe(config: Config, store: Store) -> list[Stage]:
return [describe_one(config, store, name) for name in NAMES]
return [describe_one(config, store, name) for name in SWITCHES]
+3 -2
View File
@@ -1,5 +1,5 @@
/* The Runs section of Sources: the "Run the pull now" button, and the
* switches for the two stages that ask a model.
* switches for the stages of a pull: the two that ask a model and enrich.
*
* The switches are `/api/stages`: one row per stage, drawn as the search
* rows above are (`.srow`, `.switch`), and a switch is a PUT that stores the
@@ -62,6 +62,7 @@
const DOES = {
extract: "fills in country, seniority, salary and the like from what the posting says",
summarise: "writes a short summary and tags for the best roles",
enrich: "reads the own page of the best roles whose list gave no text",
};
const el = (tag, className, text) => {
@@ -105,7 +106,7 @@
if (!stage.on) row.dataset.muted = "true";
const name = el("span", "name");
name.append(el("span", "", stage.name[0].toUpperCase() + stage.name.slice(1)),
el("span", "badge", stage.model));
...(stage.model ? [el("span", "badge", stage.model)] : []));
row.append(name, stageSwitch(stage, () => renderStages(stages)));
const sub = el("span", "sub");
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
+1 -1
View File
@@ -414,7 +414,7 @@ class StageBody(BaseModel):
@router.get("/stages")
def list_stages(store: StoreDep, config: ConfigDep) -> list[jobs.stages.Stage]:
"""The two model stages of a pull: whether each is on, whether the
"""The stages of a pull with a switch, extract, summarise and enrich: whether each is on, whether the
stored switch or config.yaml says so, its model, and who would answer."""
return jobs.stages.describe(config, store)
+23
View File
@@ -178,6 +178,29 @@ posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link.
**Helpers.** `._shared` holds what more than one host needs; a module keeps
only its own word tables and uses these:
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `table_word(table, value, log, what)` maps a host's word (stated seniority
or employment) through the module's table, without case. The table's keys
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
`permanent`) names no level and gives None; an unknown word gives None and
a debug line naming the host, so the table can grow.
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
`contract`, `temporary`, `internship`, `freelance`. Every value of a
module's `_EMPLOYMENT` table is one of these or `""`; the test in
`tests/test_readers_shared.py` holds every module to it. Seniority values
are `app.models.SENIORITIES`.
- `country_if_single(codes)` is the code when the role's places share exactly
one, else None, for a host that states a country per place.
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
headline or town line (Danish, Swedish, English).
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
is shown and embedded until `description` arrives. A reader that sees one
+59
View File
@@ -8,6 +8,7 @@ import it without either importing the other. The leading underscore keeps
from __future__ import annotations
import logging
import re
from collections.abc import Callable
from datetime import UTC, datetime
@@ -16,6 +17,8 @@ from typing import Any, Protocol
import httpx
from ._contract import Place
#: What an undeclared module's parser turns a feed into; a module that
#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`.
#: One dict per row, with the keys
@@ -192,3 +195,59 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
tail = f" per {period}" if period else ""
code = str(currency or "").strip().upper()
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
#: The employment words a row's `employment` may hold, for every host. A
#: host's table maps its own words onto these; a word that names none of them
#: (a permanent contract says nothing about hours) maps to "".
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
def place(text: str, remote: bool | None = None) -> Place:
"""A `Place` for `text`: remote when the text says the word `remote`,
otherwise as `remote` says (None when the host says nothing)."""
return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote}
def add_place(found: list[Place], entry: Place) -> None:
"""Append `entry` to `found` unless its text is empty or already there;
the host's order is kept."""
if entry["text"] and entry["text"] not in (p["text"] for p in found):
found.append(entry)
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
"""`table`'s word for a host's `value`, looked up without case. None when
the host states none, when the table maps the word to "" (a known word
that names no level, such as `experienced`), or when the table does not
know it; that last case is logged at debug level so the table can grow."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
return found or None
def country_if_single(codes: Any) -> str | None:
"""The one code among `codes` when there is exactly one, else None: a
country is stated for a role only when all its places share it."""
return next(iter(codes)) if len(codes) == 1 else None
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word a search provider's headline or
town line says about where the work is done, in Danish, Swedish and
English. A board states its own remote flag and needs none of this."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
-19
View File
@@ -1,19 +0,0 @@
"""What the search providers' headlines and town lines say about where the
work is done, in Danish, Swedish and English. Shared by the search readers;
a board states its own remote flag and needs none of this."""
from __future__ import annotations
import re
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word the text says."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
+1 -2
View File
@@ -24,8 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._words import remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
NAME = "adzuna"
LABEL = "Adzuna"
+4 -13
View File
@@ -9,14 +9,13 @@ from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__)
@@ -42,7 +41,7 @@ _API = "api.ashbyhq.com"
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
#: `employmentType` -> the app's employment words.
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern",
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
"contract": "contract", "temporary": "temporary"}
@@ -124,20 +123,12 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
for s in j.get("secondaryLocations") or []]
out: list[Place] = []
for text, remote in named:
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else remote})
add_place(out, place(text, remote))
return out
def _employment(value: Any) -> str | None:
word = str(value or "").strip()
if not word:
return None
found = _EMPLOYMENT.get(word.lower())
if found is None:
log.debug("ashby: unknown employmentType %r", word)
return found
return table_word(_EMPLOYMENT, value, log, "employmentType")
def parse(body: bytes, slug: str) -> list[Row]:
+4 -12
View File
@@ -17,14 +17,13 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, label_slug_ok, only, parse_dt
from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word
log = logging.getLogger(__name__)
@@ -39,7 +38,7 @@ READS = Reads(
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"temporary": "temporary", "intern": "intern", "internship": "intern",
"temporary": "temporary", "intern": "internship", "internship": "internship",
"freelance": "freelance"}
NAME = "bamboohr"
@@ -97,18 +96,11 @@ def _place(j: dict[str, Any]) -> str:
def _places(j: dict[str, Any]) -> list[Place]:
"""The one place BambooHR states, if it states any."""
text = _place(j)
return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \
if text else []
return [place(text)] if text else []
def _employment(j: dict[str, Any]) -> str | None:
label = str(j.get("employmentStatusLabel") or "").strip()
if not label:
return None
found = _EMPLOYMENT.get(label.lower())
if found is None:
log.debug("bamboohr: unknown employmentStatusLabel %r", label)
return found
return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel")
def parse(body: bytes, slug: str) -> list[Row]:
+5 -8
View File
@@ -24,7 +24,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
from ._shared import num, only, parse_dt, pay_line
from ._shared import num, only, parse_dt, pay_line, table_word
log = logging.getLogger(__name__)
@@ -65,8 +65,8 @@ _REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: The name of a `jobEmploymentTypes` entry -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"consultant": "contract", "temporary": "temporary", "intern": "intern",
"internship": "intern", "freelance": "freelance"}
"consultant": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
def slug_ok(slug: str) -> bool:
@@ -123,12 +123,9 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
def _employment(job: dict[str, Any]) -> str | None:
"""The first `jobEmploymentTypes` entry that names a known type."""
for entry in job.get("jobEmploymentTypes") or []:
name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip()
if not name:
continue
if found := _EMPLOYMENT.get(name.lower()):
name = ((entry or {}).get("employmentType") or {}).get("name")
if found := table_word(_EMPLOYMENT, name, log, "employment type"):
return found
log.debug("deel: unknown employment type %r", name)
return None
+2 -3
View File
@@ -44,7 +44,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of
from ._shared import Get, Page, is_web_url, only, parse_dt
from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
@@ -131,8 +131,7 @@ def _places(locations: Any) -> list[Place]:
if name and name not in names:
names.append(name)
text = ", ".join(names)
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if "remote" in text.lower() else None})
add_place(out, place(text))
return out
+7 -11
View File
@@ -27,7 +27,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt
from ._shared import Get, Page, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__)
@@ -69,9 +69,9 @@ DETAIL_QUERY = (
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: `job.employmentType` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract",
"CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern",
"TEMPORARY": "temporary", "FREELANCE": "freelance"}
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract",
"contractor": "contract", "intern": "internship", "internship": "internship",
"temporary": "temporary", "freelance": "freelance"}
def slug_ok(slug: str) -> bool:
@@ -131,8 +131,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
places: list[Place] = []
for loc in locations:
text = str(loc.get("city") or loc.get("name") or "").strip()
if text and text not in (pl["text"] for pl in places):
places.append({"text": text, "remote": loc["isRemote"]
add_place(places, {"text": text, "remote": loc["isRemote"]
if isinstance(loc.get("isRemote"), bool) else None})
job = p.get("job") or {}
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
@@ -150,11 +149,8 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
"remote": remote,
"salary_text": "",
}
if word := str(job.get("employmentType") or "").strip().upper():
if employment := _EMPLOYMENT.get(word):
row["employment"] = employment
else:
log.debug("gem: unknown employmentType %r", word)
if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"):
row["employment"] = employment
out.append(row)
return out
+4 -7
View File
@@ -15,7 +15,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
NAME = "greenhouse"
LABEL = "Greenhouse"
@@ -132,19 +132,16 @@ def _places(j: dict[str, Any]) -> list[Place]:
out: list[Place] = []
for office in j.get("offices") or []:
text = str((office or {}).get("location") or (office or {}).get("name") or "").strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
add_place(out, place(text))
if not out or not name or _PLACEHOLDER.fullmatch(name):
return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I)
else None}] if name else [])
return out or ([place(name)] if name else [])
# A name that says something the offices do not (a "Remote India" beside
# an office in Bangalore) is one more place; one they already contain,
# or that contains them, is the same place said twice.
low = name.lower()
if any(p["text"].lower() in low or low in p["text"].lower() for p in out):
return out
return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out]
return [place(name), *out]
def parse(body: bytes, slug: str) -> list[Row]:
+4 -8
View File
@@ -23,7 +23,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__)
@@ -44,7 +44,7 @@ _EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
"full-time": "full_time", "regular part time": "part_time",
"part time": "part_time", "part-time": "part_time", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
"intern": "intern", "internship": "intern", "freelance": "freelance"}
"intern": "internship", "internship": "internship", "freelance": "freelance"}
NAME = "hibob"
LABEL = "HiBob"
@@ -132,11 +132,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"salary_max": salary_max,
"currency": code,
}
kind = str(ad.get("employmentType") or "").strip()
if kind:
if employment := _EMPLOYMENT.get(kind.lower()):
row["employment"] = employment
else:
log.debug("hibob: unknown employmentType %r", kind)
if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"):
row["employment"] = employment
out.append(row)
return out
+7 -13
View File
@@ -38,7 +38,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt
from ._shared import Get, Page, add_place, only, parse_dt, place, table_word
log = logging.getLogger(__name__)
@@ -50,9 +50,9 @@ READS = Reads(
)
#: `employment_type` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract",
"CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern",
"INTERNSHIP": "intern", "FREELANCE": "freelance"}
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
"contract": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
NAME = "jibe"
LABEL = "Jibe"
@@ -120,9 +120,7 @@ def _places(j: dict[str, Any]) -> list[Place]:
out: list[Place] = []
for part in line.split(";"):
text = part.strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
add_place(out, place(text))
return out
@@ -162,11 +160,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
code = str(j.get("country_code") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
row["country"] = code
kind = str(j.get("employment_type") or "").strip()
if kind:
if employment := _EMPLOYMENT.get(kind.upper()):
row["employment"] = employment
else:
log.debug("jibe: unknown employment_type %r", kind)
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
row["employment"] = employment
out.append(row)
return out
+1 -2
View File
@@ -28,8 +28,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._words import remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex"
+9 -25
View File
@@ -25,8 +25,8 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, parse_dt
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, parse_dt, place, table_word
log = logging.getLogger(__name__)
@@ -70,7 +70,7 @@ _SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
#: The embed's `employment_type` -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "consultant": "contract",
"temporary": "temporary", "internship": "intern", "intern": "intern",
"temporary": "temporary", "internship": "internship", "intern": "internship",
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
@@ -179,22 +179,6 @@ def _published(value: Any) -> Any:
return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}")
def _mapped(table: dict[str, str], value: Any, what: str) -> str | None:
"""`table`'s word for the host's `value`; None when the host states none
or the table does not know it (logged, so the table can grow)."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("jobylon: unknown %s %r", what, word)
return found or None
def _place(text: str) -> Place:
return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}
def parse(body: bytes, slug: str) -> list[Row]:
"""The embed page -> one row per role. `posted` is the day `published_date`
names, which `detail` replaces with the exact time. `location` joins the
@@ -210,7 +194,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
continue
function = str(j.get("function") or "").strip()
places = [_place(name) for name in dict.fromkeys(
places = [place(name) for name in dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
row: Row = {
"source_id": job_id,
@@ -229,9 +213,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
if summary and summary != "None":
row["teaser"] = summary
if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"):
if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"):
row["seniority"] = seniority
if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"):
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
row["employment"] = employment
out.append(row)
return out
@@ -252,8 +236,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
else:
raise ValueError("no JobPosting on the role's page")
places = []
for place in data.get("jobLocation") or []:
address = (place or {}).get("address") or {}
for loc in data.get("jobLocation") or []:
address = (loc or {}).get("address") or {}
name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip()
if name and name not in places:
places.append(name)
@@ -263,5 +247,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
}
if places:
found["location"] = "; ".join(places)
found["places"] = [_place(name) for name in places]
found["places"] = [place(name) for name in places]
return found
+5 -11
View File
@@ -24,7 +24,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ READS = Reads(
#: text on Lever ("Permanent" is seen too and says nothing about hours).
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
"intern": "intern", "internship": "intern", "contract": "contract",
"intern": "internship", "internship": "internship", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
NAME = "lever"
@@ -115,9 +115,7 @@ def _places(categories: dict[str, Any]) -> list[Place]:
out: list[Place] = []
for name in names:
text = str(name or "").strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
add_place(out, place(text))
return out
@@ -158,11 +156,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
country = str(j.get("country") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
row["country"] = country
commitment = str(categories.get("commitment") or "").strip()
if commitment:
if employment := _EMPLOYMENT.get(commitment.lower()):
row["employment"] = employment
else:
log.debug("lever: unknown commitment %r", commitment)
if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"):
row["employment"] = employment
rows.append(row)
return rows
+3 -4
View File
@@ -35,7 +35,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of
from ._shared import Get, Page, Row, only, parse_dt
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
NAME = "oracle"
LABEL = "Oracle Recruiting Cloud"
@@ -197,8 +197,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
_text(role.get("PrimaryLocationCountry"))),
*((_text(s.get("Name")), _text(s.get("CountryCode")))
for s in secondary)):
if name and name not in (p["text"] for p in places):
places.append({"text": name, "remote": {"remote": True, "hybrid": False,
add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
"onsite": False}.get(remote or "")})
if code:
codes.add(code.upper())
@@ -213,7 +212,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
or _text(role.get("Department")) or _text(role.get("Organization"))),
"description": "",
"teaser": _plain(role.get("ShortDescriptionStr")),
"country": next(iter(codes)) if len(codes) == 1 else None,
"country": country_if_single(codes),
"remote": remote,
"salary_text": "", # the resource states no pay
})
+3 -9
View File
@@ -14,7 +14,7 @@ from collections.abc import Callable
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__)
@@ -86,12 +86,7 @@ def _salary_text(salary_info: ET.Element) -> str:
def _seniority(word: str) -> str | None:
key = word.strip().lower()
if key not in _SENIORITY:
if key:
log.debug("personio: unknown seniority %r", word)
return None
return _SENIORITY[key] or None
return table_word(_SENIORITY, word, log, "seniority")
def _employment(position: ET.Element) -> str | None:
@@ -113,8 +108,7 @@ def _places(position: ET.Element) -> list[Place]:
*(o.text for o in position.findall("additionalOffices/office"))]
for name in names:
text = (name or "").strip()
if text and text not in (p["text"] for p in found):
found.append({"text": text, "remote": True if "remote" in text.lower() else None})
add_place(found, place(text))
return found
+4 -5
View File
@@ -31,7 +31,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import name_of
from ._shared import Get, Page, Row, only, parse_dt
from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "phenom"
LABEL = "Phenom"
@@ -163,10 +163,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
if not title or not key:
continue
places: list[Place] = []
for place in (job.get("location"), *(job.get("multi_location") or [])):
if isinstance(place, str) and place.strip() \
and place.strip() not in (p["text"] for p in places):
places.append({"text": place.strip(), "remote": None})
for text in (job.get("location"), *(job.get("multi_location") or [])):
if isinstance(text, str):
add_place(places, {"text": text.strip(), "remote": None})
tail = _slug_of(title)
out.append({
"source_id": key,
+1 -2
View File
@@ -22,8 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._words import remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
NAME = "platsbanken"
LABEL = "Platsbanken"
+5 -10
View File
@@ -15,7 +15,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__)
@@ -83,12 +83,7 @@ def _salary_text(salary: dict[str, Any]) -> str:
def _seniority(code: str) -> str | None:
key = code.strip().lower()
if key not in _SENIORITY:
if key:
log.debug("recruitee: unknown experience_code %r", code)
return None
return _SENIORITY[key] or None
return table_word(_SENIORITY, code, log, "experience_code")
def _employment(code: str) -> str | None:
@@ -112,8 +107,8 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
def add(text: str) -> None:
text = text.strip()
if text and text.lower() not in _NO_PLACE and text not in (p["text"] for p in found):
found.append({"text": text, "remote": None})
if text.lower() not in _NO_PLACE:
add_place(found, {"text": text, "remote": None})
for loc in offer.get("locations") or []:
if not isinstance(loc, dict):
@@ -165,7 +160,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
"country": next(iter(codes)) if len(codes) == 1 else None,
"country": country_if_single(codes),
"seniority": _seniority(str(j.get("experience_code") or "")),
"employment": _employment(str(j.get("employment_type_code") or "")),
})
+3 -3
View File
@@ -20,7 +20,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Row, parse_dt, yearly_amount
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
log = logging.getLogger(__name__)
@@ -100,8 +100,8 @@ def parse(body: bytes, slug: str) -> list[Row]:
continue
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
places.setdefault(job_id, [])
if place and place not in (p["text"] for p in places[job_id]):
places[job_id].append(_place(place))
if place:
add_place(places[job_id], _place(place))
roles.setdefault(job_id, {
"source_id": job_id,
"title": title,
+2 -5
View File
@@ -19,7 +19,7 @@ from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
@@ -97,10 +97,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
"""The `id` of the object `job[key]` looked up in `table`; none for a word
the table maps to none, and a word it does not know is logged."""
word = str((job.get(key) or {}).get("id") or "").strip().lower()
if word and word not in table:
log.debug("smartrecruiters: unknown %s %r", key, word)
return table.get(word) or None
return table_word(table, (job.get(key) or {}).get("id"), log, key)
def rows(data: dict[str, Any], slug: str) -> list[Row]:
+3 -4
View File
@@ -59,7 +59,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
@@ -392,13 +392,12 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
addresses = _addresses(text) or [meta]
places: list[Place] = []
for address in addresses:
if (place := _location(address)) and place not in (p["text"] for p in places):
places.append({"text": place, "remote": None})
add_place(places, {"text": _location(address), "remote": None})
if places:
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
codes = {_country_code(address) for address in addresses}
if len(codes) == 1 and (code := next(iter(codes))):
if code := country_if_single(codes):
out["country"] = code
return out
+3 -4
View File
@@ -17,7 +17,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
NAME = "teamtailor"
LABEL = "Teamtailor"
@@ -129,9 +129,8 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for loc in item.findall(f"{_TT}locations/{_TT}location"):
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
country = (loc.findtext(f"{_TT}country") or "").strip()
place = city if country == city else ", ".join(p for p in (city, country) if p)
if place and place not in (p["text"] for p in places):
places.append({"text": place, "remote": remote})
text = city if country == city else ", ".join(p for p in (city, country) if p)
add_place(places, {"text": text, "remote": remote})
out.append({
"source_id": (item.findtext("guid") or url).strip(),
"title": title,
+4 -8
View File
@@ -13,7 +13,7 @@ from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
@@ -82,8 +82,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
if part and part not in parts:
parts.append(part)
text = ", ".join(parts)
if text and text not in (p["text"] for p in found):
found.append({"text": text, "remote": remote})
add_place(found, {"text": text, "remote": remote})
if code := str(loc.get("countryCode") or "").strip().upper():
codes.add(code)
return found, codes
@@ -107,9 +106,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
continue
remote = bool(j.get("remote")) or j.get("workplace") == "remote"
places, codes = _places(j, remote)
kind = str(j.get("type") or "").strip().lower()
if kind and kind not in _EMPLOYMENT:
log.debug("workable: unknown type %r", kind)
dept = j.get("department") or ""
if isinstance(dept, list):
dept = ", ".join(d for d in dept if d)
@@ -124,7 +120,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"description": "",
"remote": remote,
"salary_text": "", # Workable's list endpoint states no pay at all
"country": next(iter(codes)) if len(codes) == 1 else None,
"employment": _EMPLOYMENT.get(kind),
"country": country_if_single(codes),
"employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
})
return out
+4 -5
View File
@@ -39,7 +39,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
from ._shared import Get, Page, Row, only, parse_dt
from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__)
@@ -320,7 +320,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
out["location"] = "; ".join(p["text"] for p in places)
if len(places) == 1 and (code := _country_code(posting)):
out["country"] = code
if employment := _EMPLOYMENT.get(str(posting.get("timeType") or "").strip().lower()):
if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
out["employment"] = employment
if posted := parse_dt(posting.get("startDate")):
out["posted"] = posted
@@ -344,9 +344,8 @@ def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
(LEGO's "Billund") gets it from the role's `country`."""
found: list[Place] = []
for name in [posting.get("location"), *(posting.get("additionalLocations") or [])]:
place = _place(name, remote)
if place and place["text"] not in (p["text"] for p in found):
found.append(place)
if place := _place(name, remote):
add_place(found, place)
country = str((posting.get("country") or {}).get("descriptor") or "").strip()
if len(found) == 1 and country and country.lower() not in found[0]["text"].lower():
found[0]["text"] += f", {country}"
+1 -1
View File
@@ -141,7 +141,7 @@ def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged(
assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values())
rows = {r["source_id"]: r for r in jobylon.parse(
(FIXTURES / "jobylon-places.html").read_bytes(), "3143")}
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "intern"
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship"
def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None:
+1 -1
View File
@@ -56,7 +56,7 @@ def test_it_reads_title_company_town_date_and_teaser() -> None:
def test_it_reads_hybrid_and_remote_from_the_town_line() -> None:
from plugins.readers._words import remote_word
from plugins.readers._shared import remote_word
assert remote_word("Birkerød, Hybrid position") == "hybrid"
assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
+85
View File
@@ -0,0 +1,85 @@
"""The helpers the host modules share, and the one employment vocabulary."""
from __future__ import annotations
import logging
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import _shared # noqa: E402
from plugins.readers._shared import ( # noqa: E402
EMPLOYMENTS,
add_place,
country_if_single,
place,
remote_word,
table_word,
)
#: The module-level names a reader keeps its employment words in.
_TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE")
def test_every_employment_table_maps_into_the_shared_vocabulary() -> None:
found = 0
for name, module in readers.READERS.items():
for table_name in _TABLES:
table = getattr(module, table_name, None)
if table is None:
continue
found += 1
values = table.values() if isinstance(table, dict) else [v for _k, v in table]
# "" is a known word that names no employment ("permanent").
assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name)
assert found >= 15
def test_the_vocabulary_is_the_documented_one() -> None:
assert EMPLOYMENTS == ("full_time", "part_time", "contract", "temporary", "internship",
"freelance")
def test_place_is_remote_on_the_word_and_otherwise_as_stated() -> None:
assert place("Remote (DE)") == {"text": "Remote (DE)", "remote": True}
assert place("Berlin, remote") == {"text": "Berlin, remote", "remote": True}
assert place("Berlin") == {"text": "Berlin", "remote": None}
assert place("Berlin", False) == {"text": "Berlin", "remote": False}
assert place("Remote", False)["remote"] is True
assert place("Remotely", None)["remote"] is None
def test_add_place_skips_empty_and_repeated_texts_and_keeps_order() -> None:
found: list = []
for text in ("Berlin", "", "Oslo", "Berlin"):
add_place(found, place(text))
assert [p["text"] for p in found] == ["Berlin", "Oslo"]
def test_table_word_maps_ignores_case_and_logs_only_unknown_words(caplog) -> None:
table = {"full-time": "full_time", "permanent": ""}
log = logging.getLogger("plugins.readers.acme")
with caplog.at_level(logging.DEBUG, logger=log.name):
assert table_word(table, " Full-Time ", log, "type") == "full_time"
assert table_word(table, "Permanent", log, "type") is None
assert table_word(table, None, log, "type") is None
assert not caplog.records
assert table_word(table, "Odd", log, "type") is None
assert [r.getMessage() for r in caplog.records] == ["acme: unknown type 'Odd'"]
def test_country_if_single_needs_exactly_one_code() -> None:
assert country_if_single({"DE"}) == "DE"
assert country_if_single({"DE", "SE"}) is None
assert country_if_single(set()) is None
def test_remote_word_reads_danish_swedish_and_english() -> None:
assert remote_word("Remote, Danmark") == "remote"
assert remote_word("Birkerød, Hybrid position") == "hybrid"
assert remote_word("Hedehusene") is None
assert _shared.remote_word is remote_word
+17 -3
View File
@@ -143,7 +143,7 @@ def client(app: FastAPI) -> Iterator[TestClient]:
yield c
def test_get_stages_lists_both_with_their_source_model_and_who_answers(
def test_get_stages_lists_all_three_with_their_source_model_and_who_answers(
client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
answered_by(monkeypatch, "api")
res = client.get("/api/stages")
@@ -156,6 +156,8 @@ def test_get_stages_lists_both_with_their_source_model_and_who_answers(
{"name": "summarise", "on": False, "source": "config",
"model": "claude-haiku-4-5-20251001", "backend": "api",
"answer": "answered by the Claude API"},
{"name": "enrich", "on": True, "source": "config", "model": "", "backend": "",
"answer": "reads role pages, asks no model"},
]
@@ -164,7 +166,7 @@ def test_who_answers_is_said_for_the_api_claude_code_and_nobody(
answers = {}
for backend in ("api", "claude_code", ""):
answered_by(monkeypatch, backend)
[first, second] = client.get("/api/stages").json()
[first, second, _enrich] = client.get("/api/stages").json()
assert first["backend"] == second["backend"] == backend
answers[backend] = first["answer"]
@@ -191,6 +193,18 @@ def test_put_stores_the_setting_returns_the_same_shape_and_starts_nothing(
assert res.json()["on"] is False and store.state["stage:summarise"] == "off"
def test_the_enrich_switch_is_listed_and_stored_like_the_others(
client: TestClient, store: Store) -> None:
res = client.put("/api/stages/enrich", json={"on": False})
assert res.status_code == 200
assert res.json()["name"] == "enrich" and res.json()["on"] is False
assert store.state["stage:enrich"] == "off"
listed = {stage["name"]: stage for stage in client.get("/api/stages").json()}
assert list(listed) == ["extract", "summarise", "enrich"]
assert listed["enrich"]["on"] is False and listed["enrich"]["source"] == "stored"
def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None:
assert client.put("/api/stages/embed", json={"on": True}).status_code == 404
assert client.put("/api/stages/extract", json={"on": "sometimes"}).status_code == 422
@@ -200,7 +214,7 @@ def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> No
def test_agent_backend_off_is_named_rather_than_blamed_on_missing_keys(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: Store) -> None:
off = Config(database=str(tmp_path / "roledesk.db"), agent={"backend": "off"})
[first, _] = stages.describe(off, store)
[first, *_] = stages.describe(off, store)
assert first.backend == "" and "agent.backend" in first.answer