Merge the prep stage with the readers that hand on host words (wp-p-prep, wp-r1-raw, wp-r2-raw)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:32:15 +02:00
commit 3917b31209
117 files changed
+2452 -996

No files matched your search

+31 -1
View File
@@ -8,6 +8,7 @@
app agents list|prompt
app stages [extract|summarise on|off]
app dupes
app prep [unmapped] [--remap]
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
prep = sub.add_parser(
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
prep.add_argument("action", nargs="?", choices=["unmapped"],
help="list the host words no table maps, instead of running")
prep.add_argument("--remap", action="store_true",
help="map every role again, closed ones included")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
return run_seniority(args)
if args.command == "dupes":
return run_dupes(args)
if args.command == "prep":
return run_prep(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
def run_dupes(args: argparse.Namespace) -> int:
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
now. Prints the groups (kept roles with a copy), the copies the feed
hides and the rows whose `dup_of` changed, as JSON."""
from app.storage import Store
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
return 0
def run_prep(args: argparse.Namespace) -> int:
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
host words no table maps, per field and host, most roles first."""
from app import prep
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "unmapped":
found = store.unmapped()
if not found:
print("every host word is mapped")
for w in found:
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
return 0
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
+2 -1
View File
@@ -2,11 +2,12 @@
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
pull.py read every board and search and store the roles they yield
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
extract.py title and description into the gate columns, on Haiku
embed.py one vector per open role, for fit
score.py every open role scored for every enabled search agent
summarise.py about 60 words on each of the best roles, on Haiku
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
cascade.py pull → prep → embed → score → ... → summarise, what `TASKS`
offers, the `pull` job a schedule fires, and that schedule
stages.py whether extract and summarise run: the switch, else config.yaml
sources.py what a pull reads, changed by hand: boards and searches
+19 -13
View File
@@ -1,10 +1,13 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → enrich → embed → score → extract → score → summarise
pull → prep → embed → score → enrich → prep → embed → score → extract → score
→ summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
Nothing is alive between two tasks. A stage is addressed by its position in
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
the pull stored and folds the copies before anything reads the roles; after
`enrich` it maps the roles their own pages filled. A stage is addressed by its position in
the tuple, not its name, since `score` runs twice.
Every stage is always in the chain, and a stage that is switched off finds
@@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
text on their own pages (`app/jobs/enrich.py`), and prep, embed and score
follow for the roles it filled; when it filled none, `step` passes over all
three. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
@@ -27,7 +31,7 @@ having either way.
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
@@ -40,20 +44,20 @@ from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
"pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract",
"score", "summarise"),
"enrich": ("enrich", "prep", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
REFILL = ("prep", "embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
@@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = {
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"prep": lambda cascade: Task(kind="prep", finder=_once,
label="Mapping the roles' host words"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
@@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
"""Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
@@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any:
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise"):
for _name in ("extract", "embed", "score", "summarise", "prep"):
TASKS[_name] = partial(task, _name)
@@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, read the best roles' pages, "
label="Pull every board, then map, embed, score, read the best roles' pages, "
"extract, rescore and summarise")
+31
View File
@@ -0,0 +1,31 @@
"""`prep`: the stage that maps what a pull stored, `app.prep.run`.
In the pull cascade it follows the pull, whose settle has closed what is
gone, and maps every role the pull changed, then folds the copies across
the whole database (`Store.rebuild_dupes`, which the pull's settle used to
run). After `enrich` it maps the roles their own pages filled, since only
those are due. `solo`, so two cascades never write the same rows at once.
"""
from __future__ import annotations
import logging
from typing import Any
from app import prep
from app.config import Config
from app.storage import Store
from pipeline import NULL, RunContext, job
log = logging.getLogger(__name__)
@job("prep", "Mapping the roles' host words", solo=True, resource="cpu")
def run_prep(config: Config, store: Store, *, remap: bool = False,
context: RunContext = NULL) -> dict[str, Any]:
"""Map the due roles, every open one when the tables changed, and fold
the copies. `remap` maps every role, closed ones included."""
done = prep.run(store, remap=remap)
log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read,
done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "")
return done._asdict()
+4 -6
View File
@@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
After the closes, `Store.rebuild_dupes` recomputes which open roles are
copies of one posting, so a closed kept role hands its group on and the
roles this run added join theirs; `copies` is how many open roles it
leaves hidden as copies.
The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs.
"""
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
@@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
copies = store.rebuild_dupes().copies
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
"incomplete": incomplete, "roles": roles, "closed": closed}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
+19
View File
@@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority)
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
#: The kind of contract a posting states, as `app.prep` maps a host's word.
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
#: readers' own tests, since a plugin imports nothing from `app`.
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
"freelance"]
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
Permit = Literal["", "sponsors", "no_sponsorship"]
PERMITS: tuple[str, ...] = get_args(Permit)
@@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool:
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
def host_word(value: Any) -> str:
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
'true' or 'false', '' for nothing. What `app.prep` maps."""
if value is None:
return ""
if isinstance(value, bool):
return "true" if value else "false"
return str(value).strip()
def posted_host(raw: dict[str, Any]) -> str:
"""The host a role's dates came from, read the way `ats` and `search`
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
@@ -797,6 +814,8 @@ class Role(Model):
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
employment: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
+22
View File
@@ -0,0 +1,22 @@
"""Prep: every decision about a role that needs a rule or data across roles.
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
them in `roles.*_host`); prep maps them, once, centrally, between the pull
and embedding. One door: `from app.prep import ...`.
words.py seniority, employment and remote: the host's table, the
vocabulary itself, the shared remote words
country.py the country from the resolved places, else the host's
hosts/ one table file per reader (`hosts.TABLES`)
stage.py `run`: the stage over the due roles, `unmapped` and the fold
"""
from __future__ import annotations
from app.prep.country import country
from app.prep.hosts import TABLES
from app.prep.stage import STATE, Prepped, company, fingerprint, run
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
+55
View File
@@ -0,0 +1,55 @@
"""A role's country, from its places, else from the one its host states."""
from __future__ import annotations
from collections.abc import Iterable
from app import places
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
"""The ISO code a host's country names, in the host's spelling: a code
("DK", "dk"), or a name the gazetteer has as an alias of a country
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
country. '' for nothing stated, None for a text naming no country."""
text = (stated or "").strip()
if not text:
return ""
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
return text.upper()
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
for p in gaz.aliases.get(places.fold(text), ())
if gaz.places[p].kind == "country"
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
if len(named) == 1:
return named.pop()
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
return found.pop() if len(found) == 1 else None
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
if code == "EU":
return True
europe = gaz.by_key.get(("continent", "europe"))
place_id = gaz.by_key.get(("country", code))
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
"""The one country the role's resolved places lie in, continents aside;
Europe alone is `EU`; else, when they
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
London" and the country gate reads as Europe); else the host's stated
country (`of_host`); else ''."""
ids = list(place_ids)
# A continent beside a country is the wider unit a line adds ("Home
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
# is `EU`.
found = {gaz.country(p) for p in ids} - {""}
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
found = {"EU"}
if len(found) == 1:
return found.pop()
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
return "EU"
return of_host(gaz, host_country) or ""
+61
View File
@@ -0,0 +1,61 @@
"""Each host's own words, one module per reader, named as the reader is.
A module defines any of three dicts, keyed by the host's word lowercased:
- `SENIORITY`, values in `app.models.SENIORITIES`
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
A value of "" is a word the host uses that names no value ("Experienced",
"Permanent"): known, so it is not listed as unmapped.
A host whose employment word is a code of several parts (`permanent_full_time`,
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
after `EMPLOYMENT` with the first hit winning, and must then say how a
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
lowercased word) or "token" (it equals one of the word's parts split at `_`
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
module of this package by name, so a table is added by adding its file.
"""
from __future__ import annotations
import importlib
import pkgutil
from typing import Any
#: The dict a module names, by the field it maps.
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
#: How a `MATCH` fragment may match.
MATCH_MODES = ("substring", "token")
def load() -> dict[str, dict[str, Any]]:
"""`host -> field -> {word: value}` for every module of the package, the
words lowercased and stripped. A host without a table for a field has
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
[[fragment, value], ...]}`; a module with `MATCH` and no valid
`MATCH_MODE` fails the import by name."""
tables: dict[str, dict[str, Any]] = {}
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{__name__}.{info.name}")
for field, name in FIELDS.items():
table = getattr(module, name, None)
if isinstance(table, dict):
tables.setdefault(info.name, {})[field] = {
str(k).strip().lower(): str(v) for k, v in table.items()}
rules = getattr(module, "MATCH", None)
if rules is not None:
mode = getattr(module, "MATCH_MODE", None)
if mode not in MATCH_MODES:
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
tables.setdefault(info.name, {})["match"] = {
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
return tables
TABLES = load()
+8
View File
@@ -0,0 +1,8 @@
"""Ashby's words, mapped to the app's."""
#: `workplaceType`.
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
#: `employmentType`.
EMPLOYMENT: dict[str, str] = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
"contract": "contract", "temporary": "temporary"}
+12
View File
@@ -0,0 +1,12 @@
"""BambooHR's words, mapped to the app's."""
#: `employmentStatusLabel`. BambooHR's labels are the company's own ("Full-Time",
#: "Part-Time", "Contract", "Intern").
EMPLOYMENT: dict[str, str] = {
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"temporary": "temporary", "intern": "internship", "internship": "internship",
"freelance": "freelance"}
#: `locationType`. Anything else says nothing.
REMOTE: dict[str, str] = {"0": "onsite", "1": "remote", "2": "hybrid"}
+11
View File
@@ -0,0 +1,11 @@
"""Deel's words, mapped to the app's."""
#: `workArrangementEnum`.
REMOTE: dict[str, str] = {"on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
#: The name of a `jobEmploymentTypes` entry.
EMPLOYMENT: dict[str, str] = {
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"consultant": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
+5
View File
@@ -0,0 +1,5 @@
"""Eightfold's words, mapped to the app's."""
#: `workLocationOption`. Only "onsite" and "hybrid" were seen; the other
#: spellings are the obvious ones.
REMOTE: dict[str, str] = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
+10
View File
@@ -0,0 +1,10 @@
"""Gem's words, mapped to the app's."""
#: `job.locationType`.
REMOTE: dict[str, str] = {"in_office": "onsite", "hybrid": "hybrid", "remote": "remote"}
#: `job.employmentType`.
EMPLOYMENT: dict[str, str] = {
"full_time": "full_time", "part_time": "part_time", "contract": "contract",
"contractor": "contract", "intern": "internship", "internship": "internship",
"temporary": "temporary", "freelance": "freelance"}
+13
View File
@@ -0,0 +1,13 @@
"""HiBob's words, mapped to the app's."""
#: `employmentType`, the company's own name for it. "Permanent" says nothing
#: about hours and stays unmapped.
EMPLOYMENT: dict[str, str] = {
"regular full time": "full_time", "full time": "full_time",
"full-time": "full_time", "regular part time": "part_time",
"part time": "part_time", "part-time": "part_time", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
"intern": "internship", "internship": "internship", "freelance": "freelance"}
#: `workspaceTypeId`.
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
+7
View File
@@ -0,0 +1,7 @@
"""Jibe's words, mapped to the app's."""
#: `employment_type`.
EMPLOYMENT: dict[str, str] = {
"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
"contract": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
+21
View File
@@ -0,0 +1,21 @@
"""Jobylon's words, mapped to the app's."""
#: The embed's `experience`. Only the coarse levels are mapped; "Experienced"
#: sits between "Mid level" and the managerial levels and says too little to
#: place, so it stays unmapped.
SENIORITY: dict[str, str] = {
"student": "intern", "internship": "intern", "trainee": "intern",
"entry level": "junior", "entry-level": "junior", "junior": "junior",
"graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid",
"senior": "senior", "lead": "lead", "manager": "manager", "director": "director",
"executive": "exec", "not applicable": ""}
#: The embed's `employment_type`.
EMPLOYMENT: dict[str, str] = {
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "consultant": "contract",
"temporary": "temporary", "internship": "internship", "intern": "internship",
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
#: `workspace`.
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
+9
View File
@@ -0,0 +1,9 @@
"""Lever's words, mapped to the app's."""
#: `categories.commitment`. A commitment is free text on Lever ("Permanent" is
#: seen too and says nothing about hours).
EMPLOYMENT: dict[str, str] = {
"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
"intern": "internship", "internship": "internship", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
+5
View File
@@ -0,0 +1,5 @@
"""Oracle Recruiting Cloud's words, for prep."""
#: `WorkplaceTypeCode` -> `REMOTE`'s words. A role that states none has an
#: empty code, which stays unknown.
REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
+11
View File
@@ -0,0 +1,11 @@
"""Personio's words, for prep."""
#: `seniority`. `experienced` is what most roles say and names no level.
SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec",
"experienced": ""}
#: `employment` is the feed's `employmentType`, or, for a `permanent` one
#: (which says nothing about the hours), its `schedule`.
EMPLOYMENT = {"temporary": "temporary", "freelance": "contract", "intern": "internship",
"working_student": "internship", "trainee": "internship",
"permanent": "", "full-time": "full_time", "part-time": "part_time"}
+5
View File
@@ -0,0 +1,5 @@
"""Phenom's words, for prep."""
#: The words a page's `checkRemote` says (King's pages say "On-site"; no
#: other was seen).
REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
+13
View File
@@ -0,0 +1,13 @@
"""Pinpoint's words, for prep."""
#: `workplace_type`. Anything else says nothing.
REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`)
#: says, matched by fragment: the first fragment found in the lowercased
#: word decides, so a contract type beats the hours. The fixtures hold two
#: whole words only, so this is not turned into a whole-word table.
MATCH = [("intern", "internship"), ("freelance", "contract"), ("contract", "contract"),
("fixed_term", "temporary"), ("temporary", "temporary"),
("part_time", "part_time"), ("full_time", "full_time")]
MATCH_MODE = "substring"
+12
View File
@@ -0,0 +1,12 @@
"""Platsbanken's words, for prep."""
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
#: is in the fixtures; the others are JobTech's other labels as the site
#: writes them, and a label not here is listed as unmapped.
REMOTE: dict[str, str] = {
"arbete på plats": "onsite", "på plats": "onsite",
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
"delvis distans": "hybrid",
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
"distans": "remote",
}
+16
View File
@@ -0,0 +1,16 @@
"""Recruitee's words, for prep."""
#: `experience_code`. `experienced` names no level and is known, so it maps
#: to none.
SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid",
"experienced": ""}
#: The words of `employment_type_code` (`fulltime_permanent`), which is
#: lowercased and split at "_" and "-"; the first entry found among its words
#: decides, so a contract type beats the hours. The fixtures hold three whole
#: codes only, so this is a table of words, not whole codes.
MATCH = [("internship", "internship"), ("intern", "internship"),
("freelance", "contract"), ("contract", "contract"),
("temporary", "temporary"), ("fixed", "temporary"),
("parttime", "part_time"), ("fulltime", "full_time")]
MATCH_MODE = "token"
+9
View File
@@ -0,0 +1,9 @@
"""Rippling's words, for prep."""
#: The words of `employmentType.label` (`SALARIED_FT`), which is split at
#: "_" and the first token found here decides. The fixtures hold one whole
#: label only, so this is a table of tokens (lowercased), not whole words.
MATCH = [("ft", "full_time"), ("pt", "part_time"), ("intern", "internship"),
("internship", "internship"), ("contract", "contract"), ("contractor", "contract"),
("temp", "temporary"), ("temporary", "temporary"), ("seasonal", "temporary")]
MATCH_MODE = "token"
+11
View File
@@ -0,0 +1,11 @@
"""SmartRecruiters' words, for prep."""
#: `experienceLevel.id`. The three in the middle name no level ("associate" is
#: half of Bosch's roles) and are known, so they map to none.
SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director",
"executive": "exec", "associate": "", "mid_senior_level": "",
"not_applicable": ""}
#: `typeOfEmployment.id`.
EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary",
"contract": "contract", "part_time": "part_time", "full_time": "full_time"}
+7
View File
@@ -0,0 +1,7 @@
"""Teamtailor's words, for prep."""
#: `remoteStatus`. `none` says nothing about where the role is worked and
#: `temporary` is remote for a spell only, so both are known and stay
#: unknown rather than become "onsite" or "remote".
REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite",
"none": "", "temporary": ""}
+8
View File
@@ -0,0 +1,8 @@
"""Workable's words, for prep."""
#: `workplace`.
REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite"}
#: `type`.
EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract",
"temporary": "temporary", "internship": "internship"}
+10
View File
@@ -0,0 +1,10 @@
"""Workday's words, for prep."""
#: `remoteType`. Only some tenants send it, and each words it its own way
#: (Zendesk's "Fully Flexible", Axis' "On-Site"); a word not listed, a bare
#: "No" among them, says nothing and stays unknown.
REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
#: `timeType`.
EMPLOYMENT = {"full time": "full_time", "part time": "part_time"}
+140
View File
@@ -0,0 +1,140 @@
"""The prep stage: map what the readers handed on, then fold the copies.
A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), and its country
from the places (`app.prep.country`). When the tables changed since the
last run (`fingerprint`, kept in state as `prep:tables`), every open role
is mapped again from its stored words, with no new pull. Then the words no
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
copies across the whole database.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from typing import Any, NamedTuple
from app import places
from app.models import company_key
from app.models import company_fold
from app.prep import words
from app.prep.country import country, of_host
from app.prep.hosts import TABLES
from app.prep.words import Tables
from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes.
VERSION = 1
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
class Prepped(NamedTuple):
"""What one run did: roles read, rows written, whether every role was
mapped again, words left unmapped, and the open copies after the fold
(None when it was not run)."""
read: int
written: int
remapped: bool
unmapped: int
copies: int | None
def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
"""The key a role from `name` belongs to, as `Store.resolve_company`
has it: the company an alias names, the name's own key first when
several share the fold, else `company_key(name)`."""
own = company_key(name)
keys = folds.get(company_fold(name), ())
return own if own in keys or not keys else keys[0]
def _places(raw: Any) -> list[Any]:
try:
found = json.loads(raw) if isinstance(raw, str) else raw
except ValueError:
return []
return found if isinstance(found, list) else []
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
host = str(row["host"] or "")
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
return {
"uid": row["uid"],
"due": row["prep_due"],
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
"seniority_stated": stated,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
"remote_places": words.remote(
host, None, _places(row["raw_places"]), row["place"], tables,
title=row["title"] if row["source"] == "search" else ""),
"country": country(gaz, place_ids, row["country_host"]),
}
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or (found["remote"] and found["remote"] != row["remote"]) \
or (found["remote_places"] and not row["remote"]) \
or (found["country"] and found["country"] != row["country"]) \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
tables: Tables = TABLES) -> bool:
"""Whether no table, vocabulary or gazetteer knows this host word."""
if field == "country":
return of_host(gaz, word) is None
if field == "remote" and word in ("true", "false"):
return False
return words.lookup(field, host, word, tables) is None
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
dupes: bool = True) -> Prepped:
"""Map the due roles, or every open one when the tables changed, or
every role with `remap`; list the unmapped words; fold the copies."""
mark = fingerprint(tables)
whole = remap or store.state.get(STATE) != mark
rows = store.prep_rows("due")
if whole:
due = {r["uid"] for r in rows}
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
written = []
for row in rows:
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
if _changed(row, found):
written.append(found)
store.write_prep(written)
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
store.replace_unmapped(left)
if whole:
store.state[STATE] = mark
copies = store.rebuild_dupes().copies if dupes else None
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
copies=copies)
+131
View File
@@ -0,0 +1,131 @@
"""A host's word to the app's vocabulary: seniority, employment and remote.
Each field is looked up the same way: the host's table in `hosts`, keyed by
the word lowercased, then a word that already is a value of the vocabulary
(what readers stored before they handed on the host's words, and what a
host with the app's own words sends), then, for remote, the shared word map.
An employment word may also match a host's `MATCH` fragments. A word none
of these knows is unmapped: it maps to '' and `lookup` says so,
so the prep stage can list it.
"""
from __future__ import annotations
import re
from collections.abc import Iterable, Mapping
from typing import Any
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
from app.prep.hosts import TABLES
from app.seniority import from_title
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
Tables = Mapping[str, Mapping[str, Any]]
VOCABULARIES: dict[str, tuple[str, ...]] = {
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
#: How hosts without a table of their own write remote, after `_squash`:
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
REMOTE_WORDS: dict[str, str] = {
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
"hybrid": "hybrid",
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
"officebased": "onsite", "onpremise": "onsite",
}
def _match(rules: Mapping[str, Any], key: str) -> str | None:
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
for fragment, value in rules["rules"]:
if fragment in parts if rules["mode"] == "token" else fragment in key:
return str(value)
return None
#: Remote and hybrid in a line of Danish, Swedish or English that `remote_of`
#: does not read: "Nordhavn og mulighed for hjemmearbejde", "Arbete på distans".
#: Remote first, since "fuldt hjemmearbejde" is remote.
_REMOTE_LINE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
def _line_word(text: str) -> str:
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
if found := remote_of(text):
return found
if _REMOTE_LINE.search(text):
return "remote"
return "hybrid" if _HYBRID_LINE.search(text) else ""
def _squash(word: str) -> str:
return "".join(c for c in word.lower() if c not in " -_")
def lookup(field: str, host: str, word: str,
tables: Tables = TABLES) -> str | None:
"""The value `word` maps to for `host`, '' for a known word that names
none, None for a word nothing knows. An empty word is ''."""
key = (word or "").strip().lower()
if not key:
return ""
table = tables.get(host, {})
found = table.get(field, {}).get(key)
if found is None and field == "employment" and "match" in table:
found = _match(table["match"], key)
if found is not None:
return found
if key in VOCABULARIES[field]:
return key
if field == "remote":
return REMOTE_WORDS.get(_squash(key))
return None
def seniority(host: str, word: str, title: str,
tables: Tables = TABLES) -> tuple[str, str]:
"""`(stated, from the title)`: the host's word through its table, and
`seniority.from_title`. `roles.seniority` is the model's level, else the
stated one, else the title's (`Store.write_prep`)."""
return lookup("seniority", host, word, tables) or "", from_title(title)
def employment(host: str, word: str,
tables: Tables = TABLES) -> str:
return lookup("employment", host, word, tables) or ""
def _place_text(place: Any) -> str:
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
tables: Tables = TABLES, title: str = "") -> str:
"""How the role is worked, one of `REMOTE` or ''.
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
'false' are bools). True is remote. False says nothing more than the
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
every role that is not remote, hybrid ones included. A word goes
through the host's table, else the shared map. When the value says
nothing, the place line or the place texts are read for a word
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
`title` when given, which the stage gives for a search provider's role
only ("Backend Developer - Remote"), and last a place the reader marked
remote is remote."""
if value is True or value == "true":
return "remote"
said = value not in (None, False, "false")
if said and (found := lookup("remote", host, str(value), tables)):
return found
places = list(places)
if found := _line_word(line or "; ".join(map(_place_text, places))):
return found
if title and (found := _line_word(title)):
return found
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
return "remote"
return ""
+39 -2
View File
@@ -19,7 +19,9 @@
* Under the Follow form of Company boards, a fold lists what each reader
* declares (`/api/readers`): how its list arrives, where the text is, what
* it filters at the source, how it states places, and the robots.txt
* decision taken when it was built.
* decision taken when it was built. A second fold lists the host words no
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
* host, most roles first; it is left out when there are none.
*/
(function () {
"use strict";
@@ -320,6 +322,38 @@
$("#board-form").after(fold);
}
/** The host words prep could not map, one row per field and host. */
function renderUnmapped(words) {
if (!words.length) return;
const groups = new Map();
for (const w of words) {
const key = `${w.field} · ${w.host || "no host"}`;
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(w);
}
const fold = document.createElement("details");
fold.className = "prune";
fold.id = "unmapped";
const summary = document.createElement("summary");
summary.textContent = `Host words no table maps (${words.length})`;
const list = document.createElement("div");
list.className = "srows";
for (const [key, found] of groups) {
const row = document.createElement("div");
row.className = "srow";
const name = document.createElement("span");
name.className = "name";
name.textContent = key;
const sub = document.createElement("span");
sub.className = "sub";
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
row.append(name, sub);
list.appendChild(row);
}
fold.append(summary, list);
($("#readers") || $("#board-form")).after(fold);
}
// ------------------------------------------------------ one section shown
// Sources shows one section: the one the fragment names or contains, or the
@@ -360,7 +394,10 @@
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
if ($("#board-form")) {
api("/api/readers").then(renderReaders, () => {})
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
}
try {
await load();
} catch (err) {
+33
View File
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
"""Replace one role's own places with these texts, unresolved: what a
pull stores, for `app.prep` to resolve (`relink_uids`)."""
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
for position, text in enumerate(place_texts)])
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve the own places of these roles, writing only those that now
resolve differently. Returns `uid -> place ids` in the host's order for
every role of `uids` with places, what prep reads the country from."""
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
for start in range(0, len(uids), 5000):
chunk = json.dumps(list(uids[start:start + 5000]))
for uid, position, text, place_id in conn.execute(
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
(chunk,)):
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
found: dict[str, list[int]] = {}
changed = []
for uid, rows_of in held.items():
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
if now != rows_of:
changed.append((uid, [text for _, text in sorted(rows_of)]))
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
write_many(conn, gaz, changed)
return found
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
"""`write` for many roles in one statement each way. Returns rows written."""
+82 -2
View File
@@ -18,7 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
from app.models import (
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
)
from app.seniority import from_title
from app.storage import gazetteer
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
seniority_title TEXT NOT NULL DEFAULT '',
seniority_host TEXT NOT NULL DEFAULT '',
employment_host TEXT NOT NULL DEFAULT '',
remote_host TEXT NOT NULL DEFAULT '',
country_host TEXT NOT NULL DEFAULT '',
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
-- them into seniority_stated, employment, remote and country. prep_due is above 0
-- on a role whose words, company, title or places changed since prep last
-- mapped it (Store.upsert_role), which is what the next prep reads.
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
embedded_at TEXT NOT NULL
);
-- A host word no table of app/prep/hosts maps, per field (seniority,
-- employment, remote, country) and host: how many open roles carry it and
-- when prep first met it. Rewritten by every prep; a word added to a table
-- leaves at the next one.
CREATE TABLE IF NOT EXISTS unmapped (
field TEXT NOT NULL,
host TEXT NOT NULL,
word TEXT NOT NULL,
roles INTEGER NOT NULL DEFAULT 0,
first_seen TEXT NOT NULL,
PRIMARY KEY (field, host, word)
);
-- The string map a desk plugin is handed as self.state. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
lambda conn: _add_all_roles(conn)),
# A Python step: the host words are read out of raw's JSON.
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
"to tag; it reads the agents' reach now", target, key)
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
_V22 = """
CREATE TABLE IF NOT EXISTS unmapped (
field TEXT NOT NULL,
host TEXT NOT NULL,
word TEXT NOT NULL,
roles INTEGER NOT NULL DEFAULT 0,
first_seen TEXT NOT NULL,
PRIMARY KEY (field, host, word)
);
DELETE FROM state WHERE key = 'prep:tables';
"""
def _add_host_words(conn: sqlite3.Connection) -> None:
"""Migration 22's step. The columns and `unmapped`, and the host words
of every role filled from its `raw` where it has them. Before this,
readers mapped `seniority` and `employment` in their own tables, so what
`raw` holds there is mostly a value of the vocabulary already, which
`app.prep` takes as it is. `remote` and `country` are the host's as
given. The `prep:tables` fingerprint goes, so the first prep maps every
open role from these words. Runs twice without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
for column in _V22_COLUMNS:
if column not in columns:
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
if "prep_due" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
for statement in _statements(_V22):
conn.execute(statement)
updates = []
for uid, raw in conn.execute(
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
"OR json_extract(raw, '$.employment') IS NOT NULL "
"OR json_extract(raw, '$.remote') IS NOT NULL "
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
try:
given = json.loads(raw or "{}")
except ValueError:
continue
if isinstance(given, dict):
updates.append((*(host_word(given.get(k)) for k in (
"seniority", "employment", "remote", "country")), uid))
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
log.info("migration 22: host words from raw for %d roles", len(updates))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+140 -44
View File
@@ -22,13 +22,12 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
host_word, posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
#: says less than `extract` found keeps what `extract` found. `function` is
#: not here: it is not a board field (see the note by `keep_filled` below).
#: Nor is `seniority`, which is the title's level unless a board states one.
_RAW_TEXT = ("place", "country", "currency")
_RAW_TEXT = ("place", "currency")
#: The host's own words a reader hands on in `Item.raw`, stored as
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
#: rewritten every pull, as the host's level always was; the others are kept
#: filled, since a role's own page (`enrich`) may state what its listing
#: does not.
_HOST_WORDS = ("seniority", "employment", "remote", "country")
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
#: What prep maps from that a pull rewrites whatever it brings.
_PREP_OVERWRITTEN = ("title", "seniority_host")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
return moment.astimezone(UTC).isoformat(timespec="seconds")
def _remote(value: Any) -> str:
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
unrecognised is '' (unknown), which `extract` may fill later."""
if value is True:
return "remote"
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
return text if text in REMOTE else ""
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
"""One role's own place texts as stored, in the host's order."""
return list(dict.fromkeys(r[0] for r in conn.execute(
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
(uid,))))
def _like(text: str) -> str:
@@ -607,10 +613,19 @@ class Store:
"""Store one posting. Returns True when it was new.
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
every call, and a role seen again is open again. The raw columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
the subscription's; an empty value never overwrites a filled column.
Nothing is mapped here: that is `app.prep`, which runs after every
pull. The host's words go to `seniority_host`, `employment_host`,
`remote_host` and `country_host` (`host_word`), `company_key` is
written unresolved on a new role only, and the mapped columns
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
`remote`, `country`) are left as prep last wrote them. `prep_due`
is 1 on a new role and grows by one when the title, the company, a
host word or the place texts changed or the role reopens, which
is what the next prep reads.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
@@ -627,20 +642,14 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
when they differ from the stored ones; prep resolves them. Texts that
did not change keep their resolved rows, so the feed's Place filter
stays right between a pull and its prep. `place` is the line shown,
the texts joined when `raw` has none. Like `place`, a pull with no
place leaves them as they are. `place_fold` is written with `place`,
under the same keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
@@ -660,12 +669,11 @@ class Store:
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": self.resolve_company(company),
"company_key": company_key(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
"place_key": place_key(place),
"remote": _remote(raw.get("remote")),
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
@@ -677,30 +685,26 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
row["prep_due"] = 1
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", "teaser", *_RAW_TEXT)
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text", "seniority_stated", "seniority_title")]
"salary_text", "seniority_host")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -711,11 +715,17 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
# One more than before when what prep maps from changed, or the role
# reopens, so a prep that read the old values leaves it due
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
# what this pull brought.
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
for c in ("company", *_KEPT_HOST_WORDS)]
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -725,8 +735,11 @@ class Store:
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
gazetteer.write_texts(conn, row["uid"], place_texts)
if before is not None:
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
(row["uid"],))
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
@@ -1253,6 +1266,89 @@ class Store:
remotes)
return len(countries) + len(remotes)
# ----------------------------------------------------------------- prep
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
"""What `app.prep` maps a role from, and what it holds now: the roles
due (`prep_due` above 0), every open role, or every role."""
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
return self._rows(
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country,
json_extract(raw, '$.places') AS raw_places
FROM roles WHERE {where}""")
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve these roles' own places against the gazetteer, writing the
rows that resolve differently; `uid -> place ids` for each that has
places (`gazetteer.relink_uids`)."""
with self.transaction() as conn:
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
def company_folds(self) -> dict[str, list[str]]:
"""`fold -> company keys` of `company_aliases`, sorted: what
`resolve_company` asks one name at a time, for prep's whole batch."""
out: dict[str, list[str]] = {}
for r in self._rows("SELECT fold, company_key FROM company_aliases "
"ORDER BY fold, company_key"):
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
return out
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_places` and
`country`. `seniority` is the model's level, else the stated, else
the title's. An empty `remote` or `country` leaves the column as it
is, since `extract` fills an empty one. `remote_places`, what the
place texts say, only fills an empty column: it is a guess from a
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
changed the role since it was read. Returns the rows written."""
with self.transaction() as conn:
return conn.executemany(
"""UPDATE roles SET company_key = :company_key,
seniority_stated = :seniority_stated,
seniority_title = :seniority_title,
seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
:remote_places),
country = COALESCE(NULLIF(:country, ''), country),
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
def host_words(self) -> list[tuple[str, str, str, int]]:
"""`(field, host, word, open roles)` for every host word an open role
carries, the word lowercased: what prep checks against its tables."""
found: list[tuple[str, str, str, int]] = []
for field in ("seniority", "employment", "remote", "country"):
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
return found
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
`first_seen` of a word already listed."""
stamp = self.now()
with self.transaction() as conn:
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
"SELECT field, host, word, first_seen FROM unmapped")}
conn.execute("DELETE FROM unmapped")
conn.executemany(
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
"VALUES (?, ?, ?, ?, ?)",
[(*r, seen.get(r[:3], stamp)) for r in rows])
def unmapped(self) -> list[dict[str, Any]]:
"""The host words no table maps, by field and host, most roles first."""
return [dict(r) for r in self._rows(
"SELECT field, host, word, roles, first_seen FROM unmapped "
"ORDER BY field, host, roles DESC, word")]
# -------------------------------------------------------------- extract
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
+17
View File
@@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]:
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
class UnmappedOut(BaseModel):
"""A host word no table of `app/prep/hosts` maps (`Store.unmapped`)."""
field: str
host: str
word: str
roles: int
first_seen: str
@router.get("/prep/unmapped")
def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
"""The host words the last prep could not map, per field and host, most
roles first: what a table in `app/prep/hosts` should learn."""
return [UnmappedOut(**w) for w in store.unmapped()]
class SearchBody(BaseModel):
provider: str
name: str = ""
+6 -16
View File
@@ -178,28 +178,18 @@ posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link.
**Helpers.** `._shared` holds what more than one host needs; a module keeps
only its own word tables and uses these:
**Helpers.** `._shared` holds what more than one host needs. A module maps
no word to the app's vocabulary: it hands on the host's own words under
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
through the host's table in `app/prep/hosts/<name>.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `table_word(table, value, log, what)` maps a host's word (stated seniority
or employment) through the module's table, without case. The table's keys
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
`permanent`) names no level and gives None; an unknown word gives None and
a debug line naming the host, so the table can grow.
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
`contract`, `temporary`, `internship`, `freelance`. Every value of a
module's `_EMPLOYMENT` table is one of these or `""`; the test in
`tests/test_readers_shared.py` holds every module to it. Seniority values
are `app.models.SENIORITIES`.
- `country_if_single(codes)` is the code when the role's places share exactly
one, else None, for a host that states a country per place.
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
headline or town line (Danish, Swedish, English).
- `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers'
tests, since a plugin imports nothing from `app`.
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
+3 -37
View File
@@ -8,7 +8,6 @@ import it without either importing the other. The leading underscore keeps
from __future__ import annotations
import logging
import re
from collections.abc import Callable
from datetime import UTC, datetime
@@ -197,9 +196,9 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
#: The employment words a row's `employment` may hold, for every host. A
#: host's table maps its own words onto these; a word that names none of them
#: (a permanent contract says nothing about hours) maps to "".
#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a
#: host's employment word onto, kept for the readers' tests since a plugin
#: imports nothing from `app`. `tests/test_prep.py` holds the two equal.
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
@@ -218,36 +217,3 @@ def add_place(found: list[Place], entry: Place) -> None:
found.append(entry)
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
"""`table`'s word for a host's `value`, looked up without case. None when
the host states none, when the table maps the word to "" (a known word
that names no level, such as `experienced`), or when the table does not
know it; that last case is logged at debug level so the table can grow."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
return found or None
def country_if_single(codes: Any) -> str | None:
"""The one code among `codes` when there is exactly one, else None: a
country is stated for a role only when all its places share it."""
return next(iter(codes)) if len(codes) == 1 else None
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word a search provider's headline or
town line says about where the work is done, in Danish, Swedish and
English. A board states its own remote flag and needs none of this."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
+1 -2
View File
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "adzuna"
LABEL = "Adzuna"
@@ -124,7 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": town,
"country": country.upper(),
"remote": remote_word(title),
"posted": parse_dt(job.get("created")),
"teaser": _text(job.get("description") or ""),
"salary_min": None if predicted else job.get("salary_min"),
+6 -17
View File
@@ -15,7 +15,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -36,13 +36,6 @@ READS = Reads(
_API = "api.ashbyhq.com"
#: `workplaceType` -> `store._remote`'s words. `isRemote` is true for hybrid
#: roles too, so it is only the fallback when the role has no `workplaceType`.
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
#: `employmentType` -> the app's employment words.
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
"contract": "contract", "temporary": "temporary"}
def slug_ok(slug: str) -> bool:
@@ -113,12 +106,12 @@ def _salary_text(compensation: dict[str, Any]) -> str:
or compensation.get("scrapeableCompensationSalarySummary") or "")
def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
def _places(j: dict[str, Any], workplace: str) -> list[Place]:
"""The primary `location`, then each of `secondaryLocations`, in the host's
words. The primary is remote or an office as `workplaceType` says; a
place that names itself remote is remote."""
primary = (j.get("location") or "").strip()
named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace or ""))]
named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace.lower()))]
named += [(str((s or {}).get("location") or "").strip(), None)
for s in j.get("secondaryLocations") or []]
out: list[Place] = []
@@ -127,10 +120,6 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
return out
def _employment(value: Any) -> str | None:
return table_word(_EMPLOYMENT, value, log, "employmentType")
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
@@ -142,7 +131,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
workplace = _WORKPLACE.get(str(j.get("workplaceType") or "").strip().lower())
workplace = str(j.get("workplaceType") or "").strip()
places = _places(j, workplace)
row: Row = {
"source_id": str(j.get("id") or ""),
@@ -153,13 +142,13 @@ def parse(body: bytes, slug: str) -> list[Row]:
"posted": parse_dt(j.get("publishedAt")),
"department": (j.get("department") or j.get("team") or "").strip(),
"description": strip_html(description),
"remote": workplace or j.get("isRemote"),
"remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too
"salary_text": _salary_text(compensation),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
}
if employment := _employment(j.get("employmentType")):
if employment := str(j.get("employmentType") or "").strip():
row["employment"] = employment
rows.append(row)
return rows
+7 -20
View File
@@ -23,7 +23,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word
from ._shared import Get, label_slug_ok, only, parse_dt, place
log = logging.getLogger(__name__)
@@ -34,13 +34,6 @@ READS = Reads(
robots="read: robots.txt allows /careers (checked 2026-09-29)",
)
#: `employmentStatusLabel` -> the app's employment words. BambooHR's labels are
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"temporary": "temporary", "intern": "internship", "internship": "internship",
"freelance": "freelance"}
NAME = "bamboohr"
LABEL = "BambooHR"
PATTERNS = (
@@ -49,9 +42,6 @@ PATTERNS = (
r"https?://(?!(?:app|resources|documentation|help)\.)([a-z0-9-]+)\.bamboohr\.com",
)
#: `locationType` -> `store._remote`'s words. Anything else says nothing.
_REMOTE = {"0": "onsite", "1": "remote", "2": "hybrid"}
#: Sent on the detail read: without it the same URL answers the page's HTML.
_JSON = {"Accept": "application/json"}
@@ -76,10 +66,11 @@ def feed_url(slug: str, ask: Ask) -> str:
return f"https://{host(slug)}/careers/list"
def _remote(j: dict[str, Any]) -> str | None:
def _remote(j: dict[str, Any]) -> bool | str | None:
"""`isRemote` when true, else `locationType` as the host writes it ("0", "1", "2")."""
if j.get("isRemote") is True:
return "remote"
return _REMOTE.get(str(j.get("locationType") if j.get("locationType") is not None else ""))
return True
return None if j.get("locationType") is None else str(j["locationType"]).strip() or None
def _place(j: dict[str, Any]) -> str:
@@ -99,10 +90,6 @@ def _places(j: dict[str, Any]) -> list[Place]:
return [place(text)] if text else []
def _employment(j: dict[str, Any]) -> str | None:
return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel")
def parse(body: bytes, slug: str) -> list[Row]:
"""`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A
role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`,
@@ -136,7 +123,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"remote": _remote(j),
"salary_text": "", # the list states no pay; the detail's `compensation` was null
}
if employment := _employment(j):
if employment := str(j.get("employmentStatusLabel") or "").strip():
row["employment"] = employment
out.append(row)
return out
@@ -160,6 +147,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if place := _place(opening):
out["location"] = place
out["places"] = _places(opening)
if remote := _remote(opening):
if (remote := _remote(opening)) is not None:
out["remote"] = remote
return out
+5 -14
View File
@@ -24,7 +24,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
from ._shared import num, only, parse_dt, pay_line, table_word
from ._shared import num, only, parse_dt, pay_line
log = logging.getLogger(__name__)
@@ -59,15 +59,6 @@ _PUSH = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)')
_ROW = re.compile(rb"([0-9a-f]*):")
_TEXT = re.compile(rb"T([0-9a-f]+),")
#: Deel's `workArrangementEnum` -> `store._remote`'s words.
_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: The name of a `jobEmploymentTypes` entry -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"consultant": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
def slug_ok(slug: str) -> bool:
return bool(_SLUG.fullmatch(slug))
@@ -121,10 +112,10 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
def _employment(job: dict[str, Any]) -> str | None:
"""The first `jobEmploymentTypes` entry that names a known type."""
"""The name of the first `jobEmploymentTypes` entry that has one."""
for entry in job.get("jobEmploymentTypes") or []:
name = ((entry or {}).get("employmentType") or {}).get("name")
if found := table_word(_EMPLOYMENT, name, log, "employment type"):
if found := str(name or "").strip():
return found
return None
@@ -155,7 +146,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
description = p.get("richtextDescription") or ""
if description.startswith("$"):
description = stream.get(description[1:], "")
arrangement = str(job.get("workArrangementEnum") or "")
arrangement = str(job.get("workArrangementEnum") or "").strip()
remote = {"REMOTE": True, "HYBRID": False, "ON_SITE": False}.get(arrangement)
row: Row = {
"source_id": post_id,
@@ -166,7 +157,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
"department": ", ".join(dict.fromkeys(departments)),
"description": html.unescape(strip_html(description)),
"remote": _REMOTE.get(str(job.get("workArrangementEnum") or "")),
"remote": arrangement or None,
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
pay.get("currencyIsoCode"), "")
if p.get("isCompensationVisible") else ""),
+1 -5
View File
@@ -70,10 +70,6 @@ _NAME = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9]
#: Names a private network uses, never a company's career site.
_LOCAL = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
#: `workLocationOption` -> `store._remote`'s words. Only "onsite" and "hybrid"
#: were seen; the other spellings are the obvious ones.
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
def parts(slug: str) -> tuple[str, str]:
"""`(host, domain)` of a slug, lowercased."""
@@ -162,7 +158,7 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
"posted": parse_dt(p.get("creationTs") or p.get("postedTs")),
"department": str(p.get("department") or "").strip(),
"description": "",
"remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()),
"remote": str(p.get("workLocationOption") or "").strip() or None,
"salary_text": "",
})
return out
+4 -12
View File
@@ -27,7 +27,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, add_place, only, parse_dt, table_word
from ._shared import Get, Page, add_place, only, parse_dt
log = logging.getLogger(__name__)
@@ -65,14 +65,6 @@ DETAIL_QUERY = (
"job { id locationType employmentType requisitionId teamDisplayName "
"department { id extId name } } compensationHtml } }")
#: `job.locationType` -> `store._remote`'s words.
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: `job.employmentType` -> the app's employment words.
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract",
"contractor": "contract", "intern": "internship", "internship": "internship",
"temporary": "temporary", "freelance": "freelance"}
def slug_ok(slug: str) -> bool:
return bool(_BOARD.fullmatch(slug))
@@ -134,9 +126,9 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
add_place(places, {"text": text, "remote": loc["isRemote"]
if isinstance(loc.get("isRemote"), bool) else None})
job = p.get("job") or {}
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
remote: bool | str | None = str(job.get("locationType") or "").strip() or None
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
remote = "remote"
remote = True
row: Row = {
"source_id": ext_id,
"title": title,
@@ -149,7 +141,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
"remote": remote,
"salary_text": "",
}
if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"):
if employment := str(job.get("employmentType") or "").strip():
row["employment"] = employment
out.append(row)
return out
+3 -14
View File
@@ -23,7 +23,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
log = logging.getLogger(__name__)
@@ -38,21 +38,10 @@ READS = Reads(
"app's index page with no rules (checked 2026-09-29)",
)
#: `employmentType`, the company's own name for it -> the app's employment
#: words. "Permanent" says nothing about hours and stays unmapped.
_EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
"full-time": "full_time", "regular part time": "part_time",
"part time": "part_time", "part-time": "part_time", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
"intern": "internship", "internship": "internship", "freelance": "freelance"}
NAME = "hibob"
LABEL = "HiBob"
PATTERNS = (r"https?://([a-z0-9-]+)\.careers\.hibob\.com",)
#: HiBob's `workspaceTypeId` -> `store._remote`'s words.
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
#: The texts of an ad, in the order the ad shows them.
_SECTIONS = ("description", "responsibilities", "requirements", "benefits")
@@ -126,13 +115,13 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"posted": parse_dt(ad.get("publishedAt")),
"department": str(ad.get("department") or "").strip(),
"description": " ".join(html.unescape(text).split()),
"remote": _REMOTE.get(str(ad.get("workspaceTypeId") or "").strip().lower()),
"remote": str(ad.get("workspaceTypeId") or "").strip() or None,
"salary_text": pay_line(lo, hi, currency, period),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": code,
}
if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"):
if employment := str(ad.get("employmentType") or "").strip():
row["employment"] = employment
out.append(row)
return out
+4 -9
View File
@@ -38,7 +38,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, add_place, only, parse_dt, place, table_word
from ._shared import Get, Page, add_place, only, parse_dt, place
log = logging.getLogger(__name__)
@@ -49,11 +49,6 @@ READS = Reads(
robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
)
#: `employment_type` -> the app's employment words.
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
"contract": "contract", "temporary": "temporary", "intern": "internship",
"internship": "internship", "freelance": "freelance"}
NAME = "jibe"
LABEL = "Jibe"
PATTERNS = (
@@ -157,10 +152,10 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
}
# One code, for a role at one place: over several countries it would
# gate the others by the site's own.
code = str(j.get("country_code") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
code = str(j.get("country_code") or "").strip()
if len(places) == 1 and code:
row["country"] = code
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
if employment := str(j.get("employment_type") or "").strip():
row["employment"] = employment
out.append(row)
return out
+2 -5
View File
@@ -28,7 +28,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, place
from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex"
@@ -131,9 +131,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
town = _text(area.group(1)) if area else decoded["area"]
remote = remote_word(f"{town} {title}")
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
if town else []
places: list[Place] = [place(town)] if town else []
out.append({
"source_id": item["id"],
"title": title or full_title,
@@ -142,7 +140,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": town,
"country": "DK",
"remote": remote,
"posted": parse_dt(item["published"]),
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
})
+4 -22
View File
@@ -26,7 +26,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, parse_dt, place, table_word
from ._shared import Get, parse_dt, place
log = logging.getLogger(__name__)
@@ -58,28 +58,10 @@ _SLUG = re.compile(r"[0-9]{1,12}")
#: from.
_ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?")
#: The embed's `experience` -> `app.models.SENIORITIES`. Only the coarse levels
#: are mapped; "Experienced" sits between "Mid level" and the managerial
#: levels and says too little to place, so it stays unmapped and is logged.
_SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
"entry level": "junior", "entry-level": "junior", "junior": "junior",
"graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid",
"senior": "senior", "lead": "lead", "manager": "manager", "director": "director",
"executive": "exec", "not applicable": ""}
#: The embed's `employment_type` -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "consultant": "contract",
"temporary": "temporary", "internship": "internship", "intern": "internship",
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_TRAILING_COMMA = re.compile(r",(\s*[\]}])")
#: Jobylon's `workspace` -> `store._remote`'s words.
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
#: The English month names, by their first three letters, in the spellings
#: `published_date` uses ("Sept. 25, 2026").
_MONTHS = {m: i for i, m in enumerate(
@@ -205,7 +187,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"posted": _published(j.get("published_date")),
"department": "" if function == "All functions" else function,
"description": "",
"remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()),
"remote": str(j.get("workspace") or "").strip() or None,
"salary_text": "", # the embed states no pay
}
# `summary` is the text the company gave the listing, the string
@@ -213,9 +195,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
if summary and summary != "None":
row["teaser"] = summary
if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"):
if seniority := str(j.get("experience") or "").strip():
row["seniority"] = seniority
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
if employment := str(j.get("employment_type") or "").strip():
row["employment"] = employment
out.append(row)
return out
+4 -12
View File
@@ -17,14 +17,13 @@ from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -36,13 +35,6 @@ READS = Reads(
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `categories.commitment` -> the app's employment words. A commitment is free
#: text on Lever ("Permanent" is seen too and says nothing about hours).
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
"intern": "internship", "internship": "internship", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
NAME = "lever"
LABEL = "Lever"
RANK = 20
@@ -153,10 +145,10 @@ def parse(body: bytes, slug: str) -> list[Row]:
}
# `country` is one code, for a role at one place; over several it would
# gate the others by the first's country.
country = str(j.get("country") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
country = str(j.get("country") or "").strip()
if len(places) == 1 and country:
row["country"] = country
if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"):
if employment := str(categories.get("commitment") or "").strip():
row["employment"] = employment
rows.append(row)
return rows
+8 -18
View File
@@ -35,7 +35,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "oracle"
LABEL = "Oracle Recruiting Cloud"
@@ -55,7 +55,7 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="ids", remote="one"),
fields=Fields(posted="trusted", remote="stated", country="stated"),
fields=Fields(posted="trusted", remote="stated"),
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
"(checked 2026-09-29)"),
)
@@ -71,11 +71,6 @@ PAGE_SIZE = 200
_HOST = r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu)"
_SLUG = re.compile(rf"({_HOST})/([A-Za-z0-9_-]+)", re.IGNORECASE)
#: `WorkplaceTypeCode` -> `store._remote`'s words. A role that states none
#: has an empty code, which stays unknown.
_REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
def _parts(slug: str) -> tuple[str, str]:
"""`(host, site number)` of a slug that passed `slug_ok`."""
host, _, site = slug.partition("/")
@@ -189,18 +184,14 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
# The id goes into a URL, and a listing is external content.
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
continue
remote = _REMOTE.get(_text(role.get("WorkplaceTypeCode")).lower())
remote = _text(role.get("WorkplaceTypeCode")) or None
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
places: list[Place] = []
codes: set[str] = set()
secondary = [s for s in role.get("secondaryLocations") or [] if isinstance(s, dict)]
for name, code in ((_text(role.get("PrimaryLocation")),
_text(role.get("PrimaryLocationCountry"))),
*((_text(s.get("Name")), _text(s.get("CountryCode")))
for s in secondary)):
add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
"onsite": False}.get(remote or "")})
if code:
codes.add(code.upper())
for name in (_text(role.get("PrimaryLocation")),
*(_text(s.get("Name")) for s in secondary)):
add_place(places, {"text": name, "remote": (
True if (remote or "").lower() == "ora_remote" else False if at_office else None)})
out.append({
"source_id": role_id,
"title": title,
@@ -212,7 +203,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
or _text(role.get("Department")) or _text(role.get("Organization"))),
"description": "",
"teaser": _plain(role.get("ShortDescriptionStr")),
"country": country_if_single(codes),
"remote": remote,
"salary_text": "", # the resource states no pay
})
+9 -30
View File
@@ -7,16 +7,13 @@ Personio.
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from collections.abc import Callable
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__)
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
NAME = "personio"
LABEL = "Personio"
@@ -30,19 +27,6 @@ READS = Reads(
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `seniority` -> the app's levels. `experienced` is what most roles say and
#: names no level, so it is known and maps to none; a word not listed is
#: logged at debug level so the table can grow.
_SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec",
"experienced": ""}
#: `employmentType` when it is not the usual permanent contract, to the app's
#: employment words; `permanent` says nothing about the hours, so `schedule`
#: decides then.
_CONTRACT = {"temporary": "temporary", "freelance": "contract", "intern": "internship",
"working_student": "internship", "trainee": "internship"}
_SCHEDULE = {"full-time": "full_time", "part-time": "part_time"}
#: Personio's `salaryInformation/type` spellings, to the plain period word
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
@@ -85,18 +69,14 @@ def _salary_text(salary_info: ET.Element) -> str:
salary_info.findtext("currencyCode"), period)
def _seniority(word: str) -> str | None:
return table_word(_SENIORITY, word, log, "seniority")
def _employment(position: ET.Element) -> str | None:
contract = (position.findtext("employmentType") or "").strip().lower()
schedule = (position.findtext("schedule") or "").strip().lower()
if contract in _CONTRACT:
return _CONTRACT[contract]
if contract and contract != "permanent":
log.debug("personio: unknown employmentType %r", contract)
return _SCHEDULE.get(schedule)
"""The host's word for the contract, or, for a `permanent` one (which says
nothing about the hours), the `schedule`."""
contract = (position.findtext("employmentType") or "").strip()
schedule = (position.findtext("schedule") or "").strip()
if contract.lower() in ("", "permanent") and schedule:
return schedule
return contract or None
def _places(position: ET.Element) -> list[Place]:
@@ -148,8 +128,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"posted": parse_dt(position.findtext("createdAt")),
"department": (position.findtext("department") or "").strip(),
"description": strip_html(description),
"remote": any(p["remote"] for p in places) or None,
"seniority": _seniority(position.findtext("seniority") or ""),
"seniority": (position.findtext("seniority") or "").strip() or None,
"employment": _employment(position),
"salary_text": salary_text,
"salary_min": salary_min,
+3 -4
View File
@@ -188,10 +188,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
#: Where a page assigns its state, and the words its `checkRemote` says that
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
#: Where a page assigns its state. Its `checkRemote` says "On-site" on King's
#: pages; no other word was seen.
_STATE = "phApp.ddo = "
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
def _job_posting(body: str) -> dict[str, Any]:
@@ -243,6 +242,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
raise ValueError("no description on the role's page")
if posted := parse_dt(posting.get("datePosted")):
out["posted"] = posted
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
if remote := _text(job.get("checkRemote")):
out["remote"] = remote
return out
+4 -26
View File
@@ -18,7 +18,6 @@ from __future__ import annotations
import html
import json
import logging
from collections.abc import Callable
from typing import Any
@@ -27,8 +26,6 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Reads
from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
log = logging.getLogger(__name__)
NAME = "pinpoint"
LABEL = "Pinpoint"
PATTERNS = (r"https?://([a-z0-9-]+)\.pinpointhq\.com",)
@@ -39,9 +36,6 @@ READS = Reads(
robots="read: robots.txt allows / (checked 2026-09-29)",
)
#: `workplace_type` -> `store._remote`'s words. Anything else says nothing.
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
#: `compensation_frequency` -> the multiplier that makes a figure yearly. The
#: spellings are a guess, since no board read so far states pay; a frequency
#: not listed leaves the yearly columns empty, and `salary_text` still says it.
@@ -51,12 +45,6 @@ _YEARLY = {"annually": 1, "annual": 1, "yearly": 1, "year": 1, "monthly": 12, "m
_PERIOD = {"annually": "year", "annual": "year", "yearly": "year", "monthly": "month",
"weekly": "week", "daily": "day", "hourly": "hour"}
#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`) says,
#: the first fragment found deciding, so a contract type beats the hours.
_EMPLOYMENT = (("intern", "internship"), ("freelance", "contract"), ("contract", "contract"),
("fixed_term", "temporary"), ("temporary", "temporary"),
("part_time", "part_time"), ("full_time", "full_time"))
#: The sections of a posting's text, in the order the page shows them: the
#: description, then these fields, each with its own heading in `<field>_header`.
_SECTIONS = ("key_responsibilities", "skills_knowledge_expertise", "benefits")
@@ -112,16 +100,6 @@ def _place(location: dict[str, Any]) -> str:
return ", ".join(parts)
def _employment(code: str) -> str | None:
code = code.strip().lower()
for fragment, value in _EMPLOYMENT:
if fragment in code:
return value
if code:
log.debug("pinpoint: unknown employment_type %r", code)
return None
def _pay(j: dict[str, Any]) -> tuple[int | None, int | None, str, str]:
"""`(salary_min, salary_max, currency, salary_text)` of a posting whose
pay is shown (`compensation_visible`); the free-text `compensation` is the
@@ -154,21 +132,21 @@ def parse(body: bytes, slug: str) -> list[Row]:
continue
job = j.get("job") or {}
low, high, currency, salary_text = _pay(j)
workplace = str(j.get("workplace_type") or "").strip().lower()
workplace = str(j.get("workplace_type") or "").strip()
place = _place(j.get("location") or {})
out.append({
"source_id": source_id,
"title": title,
"places": [{"text": place, "remote": {"remote": True, "onsite": False,
"on_site": False, "hybrid": False}.get(workplace)}]
"on_site": False, "hybrid": False}.get(workplace.lower())}]
if place else [],
"location": place,
"url": url,
"posted": None, # only the posting's own page has a date
"department": str((job.get("department") or {}).get("name") or "").strip(),
"description": _text(j),
"remote": _REMOTE.get(workplace),
"employment": _employment(str(j.get("employment_type") or "")),
"remote": workplace or None,
"employment": str(j.get("employment_type") or "").strip() or None,
"salary_text": salary_text,
"salary_min": low,
"salary_max": high,
+5 -5
View File
@@ -7,8 +7,8 @@ Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`.
above 2000 is refused, so a search reads 2,100 results at most. It states
`total`. A hit has the whole ad's text, so the list has the full text. An ad
may name several workplaces: `workplace_addresses` holds one entry each, and
every one is a place of the role. `workplace_model` is mostly empty, so
`remote` comes from the headline and the model together.
every one is a place of the role. `workplace_model` is mostly empty; its
label is passed on as `remote`, in the host's words.
"""
from __future__ import annotations
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "platsbanken"
LABEL = "Platsbanken"
@@ -113,7 +113,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
places = [place for address in addresses if isinstance(address, dict)
if (place := _place(address))]
model = (hit.get("workplace_model") or {}).get("label") or ""
model = str((hit.get("workplace_model") or {}).get("label") or "").strip()
out.append({
"source_id": str(hit.get("id") or ""),
"title": title,
@@ -122,7 +122,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": places[0]["text"] if places else "",
"country": "SE",
"remote": remote_word(f"{title} {model}"),
"remote": model or None,
"posted": parse_dt(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
})
+8 -43
View File
@@ -8,16 +8,13 @@ Recruitee.
from __future__ import annotations
import json
import logging
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__)
from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
@@ -27,23 +24,11 @@ PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
fields=Fields(pay="figures", posted="trusted", remote="stated",
seniority="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `experience_code` -> the app's levels. `experienced` names no level and is
#: known, so it maps to none; a word not listed is logged at debug level.
_SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid",
"experienced": ""}
#: The words of `employment_type_code` (`fulltime_permanent`), the first that
#: is found deciding, so a contract type beats the hours.
_EMPLOYMENT = (("internship", "internship"), ("intern", "internship"),
("freelance", "contract"), ("contract", "contract"),
("temporary", "temporary"), ("fixed", "temporary"),
("parttime", "part_time"), ("fulltime", "full_time"))
#: What the board writes as a place for a role that is worked anywhere.
_NO_PLACE = ("remote job",)
@@ -82,28 +67,13 @@ def _salary_text(salary: dict[str, Any]) -> str:
salary.get("currency"), period)
def _seniority(code: str) -> str | None:
return table_word(_SENIORITY, code, log, "experience_code")
def _employment(code: str) -> str | None:
words = set(code.strip().lower().replace("-", "_").split("_"))
for word, value in _EMPLOYMENT:
if word in words:
return value
if code.strip():
log.debug("recruitee: unknown employment_type_code %r", code)
return None
def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
"""Every place of an offer and the countries' codes among them. `locations`
def _places(offer: dict[str, Any]) -> list[Place]:
"""Every place of an offer. `locations`
lists them (city, state, country), `location` is the first as one line and
is the fallback for an offer that lists none. The board writes "Remote
job" where an offer has no place: that is not one, and a remote offer with
no other place has the place "Remote"."""
found: list[Place] = []
codes: set[str] = set()
def add(text: str) -> None:
text = text.strip()
@@ -119,15 +89,11 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
if part and part not in parts:
parts.append(part)
add(", ".join(parts))
if code := str(loc.get("country_code") or "").strip().upper():
codes.add(code)
if not found:
add(str(offer.get("location") or offer.get("city") or ""))
if code := str(offer.get("country_code") or "").strip().upper():
codes.add(code)
if not found and offer.get("remote"):
found.append({"text": "Remote", "remote": True})
return found, codes
return found
def parse(body: bytes, slug: str) -> list[Row]:
@@ -145,7 +111,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
places, codes = _places(j)
places = _places(j)
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -160,8 +126,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
"country": country_if_single(codes),
"seniority": _seniority(str(j.get("experience_code") or "")),
"employment": _employment(str(j.get("employment_type_code") or "")),
"seniority": str(j.get("experience_code") or "").strip() or None,
"employment": str(j.get("employment_type_code") or "").strip() or None,
})
return rows
+3 -19
View File
@@ -12,7 +12,6 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
@@ -22,8 +21,6 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
log = logging.getLogger(__name__)
NAME = "rippling"
LABEL = "Rippling"
KEEP_CASE = True
@@ -46,11 +43,6 @@ _SITE = "ats.rippling.com"
#: whole; it is also what discovery's patterns take.
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
#: The words of `employmentType.label` (`SALARIED_FT`), split at "_".
_EMPLOYMENT = {"FT": "full_time", "PT": "part_time", "INTERN": "internship",
"INTERNSHIP": "internship", "CONTRACT": "contract", "CONTRACTOR": "contract",
"TEMP": "temporary", "TEMPORARY": "temporary", "SEASONAL": "temporary"}
#: A role's id in the board's own links.
_UUID = re.compile(r"[0-9A-Za-z-]{8,64}")
@@ -86,8 +78,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
role and place, so a role in three cities is three entries with one
`uuid` (Rippling's own board: 638 entries, 326 roles); the rows merge
them, the places kept as a list and joined with "; " for `location`. The entry has no description, date
or pay: `detail` reads them. `remote` is True only when every place is a
"Remote (...)" one, since a label says nothing else about the work."""
or pay: `detail` reads them."""
data = json.loads(body)
if not isinstance(data, list):
raise ValueError("not a Rippling board: the answer is not a list")
@@ -114,19 +105,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
for job_id, row in roles.items():
row["places"] = places[job_id]
row["location"] = "; ".join(p["text"] for p in places[job_id])
row["remote"] = True if places[job_id] and all(
p["remote"] for p in places[job_id]) else None
return list(roles.values())
def _employment(kind: Any) -> str | None:
label = str((kind or {}).get("label") or "").strip().upper() if isinstance(kind, dict) else ""
for word in label.split("_"):
if word in _EMPLOYMENT:
return _EMPLOYMENT[word]
if label:
log.debug("rippling: unknown employmentType %r", label)
return None
"""The host's `employmentType.label` (`SALARIED_FT`), as given."""
return (str(kind.get("label") or "").strip() or None) if isinstance(kind, dict) else None
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
+7 -23
View File
@@ -13,15 +13,12 @@ ignores a remote parameter, so remote is not filtered.
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "smartrecruiters"
LABEL = "SmartRecruiters"
@@ -41,17 +38,6 @@ READS = Reads(
_API = "api.smartrecruiters.com"
#: `experienceLevel.id` -> the app's levels. The three in the middle name no
#: level ("associate" is half of Bosch's roles) and are known, so they map to
#: none; an id not listed is logged at debug level.
_SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director",
"executive": "exec", "associate": "", "mid_senior_level": "",
"not_applicable": ""}
#: `typeOfEmployment.id` -> the app's employment words.
_EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary",
"contract": "contract", "part_time": "part_time", "full_time": "full_time"}
#: The page size asked for, which is also the most the host serves per page.
PAGE_SIZE = 100
@@ -94,10 +80,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
"""The `id` of the object `job[key]` looked up in `table`; none for a word
the table maps to none, and a word it does not know is logged."""
return table_word(table, (job.get(key) or {}).get("id"), log, key)
def _word(job: dict[str, Any], key: str) -> str | None:
"""The `id` of the object `job[key]`, as given."""
return str((job.get(key) or {}).get("id") or "").strip() or None
def rows(data: dict[str, Any], slug: str) -> list[Row]:
@@ -122,7 +107,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
loc_text = ", ".join(part.strip() for part in (loc_text or "").split(",")
if part.strip())
place: Place = {"text": loc_text, "remote": location.get("remote")}
country = str(location.get("country") or "").strip().upper()
out.append({
"source_id": job_id,
"title": title,
@@ -134,8 +118,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"description": "",
"remote": location.get("remote"),
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
"country": country if len(country) == 2 else None,
"seniority": _word(j, "experienceLevel", _SENIORITY),
"employment": _word(j, "typeOfEmployment", _EMPLOYMENT),
"country": str(location.get("country") or "").strip() or None,
"seniority": _word(j, "experienceLevel"),
"employment": _word(j, "typeOfEmployment"),
})
return out
+3 -19
View File
@@ -59,7 +59,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
@@ -86,7 +86,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", text="detail", complete="stated", places="many",
filters=Filters(country="one"),
fields=Fields(posted="trusted", country="stated"),
fields=Fields(posted="trusted"),
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
)
@@ -342,19 +342,6 @@ def _posted(value: str) -> datetime | None:
return parse_dt(value)
def _country_code(meta: dict[str, str]) -> str:
"""The address's country as an ISO code: `addressCountry`, or the code that
`_place` reads in a `streetAddress`; '' for neither."""
code = meta.get("addressCountry", "").strip().upper()
if len(code) == 2:
return code
parts = [part.strip() for part in meta.get("streetAddress", "").split(",")]
for i in range(len(parts) - 1, 0, -1):
if len(parts) - i <= 2 and parts[i] in _COUNTRIES:
return parts[i]
return ""
def _location(meta: dict[str, str]) -> str:
"""The address as "City, Country". Vestas and Demant state
`addressLocality` and `addressCountry` (a code), the other sites one
@@ -373,7 +360,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""A role's own page: `description` from the block marked
`itemprop="description"`, which Novo Nordisk's pages have without the
microdata around it, and, from a page that has the `JobPosting` microdata,
`posted` (`datePosted`) and every place with the country's code. A page that
`posted` (`datePosted`) and every place. A page that
is not a role's (the site answers 200 for a closed one with its search) has
no description block and gives an empty answer."""
text = get(row["url"]).decode("utf-8", "replace")
@@ -396,9 +383,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if places:
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
codes = {_country_code(address) for address in addresses}
if code := country_if_single(codes):
out["country"] = code
return out
+1 -7
View File
@@ -51,12 +51,6 @@ _HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
_TT = "{https://teamtailor.com/locations}"
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
#: about where the role is worked and `temporary` is remote for a spell only,
#: so both stay unknown rather than become "onsite" or "remote".
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
def site(slug: str) -> str:
"""The host a career site is served from: `acme` is acme.teamtailor.com,
and a slug with a dot is a site on the company's own domain,
@@ -141,7 +135,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
"department": ((item.findtext(f"{_TT}department") or "").strip()
or (item.findtext(f"{_TT}role") or "").strip()),
"description": html.unescape(strip_html(item.findtext("description") or "")),
"remote": _REMOTE.get(status),
"remote": status or None,
"salary_text": "", # Teamtailor's feed states no pay at all
})
return out
+10 -21
View File
@@ -8,14 +8,11 @@ board with roles was seen; jobhunt-hq did not read Workable.
from __future__ import annotations
import json
import logging
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
NAME = "workable"
LABEL = "Workable"
@@ -28,17 +25,12 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=10, text="none", complete="stated",
places="many",
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
_API = "apply.workable.com"
#: `type` -> the app's employment words; a type not listed is logged at debug
#: level so the table can grow.
_EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract",
"temporary": "temporary", "internship": "internship"}
# No probe: Workable's API sits behind a rate limit that a run of guesses
# trips, and a Workable site links its board plainly anyway.
@@ -67,13 +59,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
"""Every place of a role and the countries' codes among them. `locations`
def _places(job: dict[str, Any], remote: bool) -> list[Place]:
"""Every place of a role. `locations`
lists them (city, region, country; `hidden` ones are stated too), and
`location` is the first alone, the fallback for a role that lists none.
A place is remote when the role is."""
found: list[Place] = []
codes: set[str] = set()
listed = [loc for loc in job.get("locations") or [] if isinstance(loc, dict)]
for loc in listed or [job.get("location") or {}]:
parts: list[str] = []
@@ -83,9 +74,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
parts.append(part)
text = ", ".join(parts)
add_place(found, {"text": text, "remote": remote})
if code := str(loc.get("countryCode") or "").strip().upper():
codes.add(code)
return found, codes
return found
def rows(data: dict[str, Any], slug: str) -> list[Row]:
@@ -104,8 +93,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
remote = bool(j.get("remote")) or j.get("workplace") == "remote"
places, codes = _places(j, remote)
workplace = str(j.get("workplace") or "").strip()
remote = bool(j.get("remote")) or workplace == "remote"
places = _places(j, remote)
dept = j.get("department") or ""
if isinstance(dept, list):
dept = ", ".join(d for d in dept if d)
@@ -118,9 +108,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"posted": parse_dt(j.get("published")),
"department": dept.strip(),
"description": "",
"remote": remote,
"remote": workplace or j.get("remote"),
"salary_text": "", # Workable's list endpoint states no pay at all
"country": country_if_single(codes),
"employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
"employment": str(j.get("type") or "").strip() or None,
})
return out
+18 -30
View File
@@ -29,7 +29,6 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
@@ -39,9 +38,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "workday"
LABEL = "Workday"
@@ -50,7 +47,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many", filters=Filters(country="ids", remote="ids"),
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
)
@@ -167,7 +164,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
"""The tenant's host ids, read off one page of one role: `country` maps
an ISO code to `<facetParameter>=<id>` in the facet whose values are
countries (most of them, `_COUNTRY_SHARE`; the one with most countries
when two are), and `remote` maps each facet value that `_REMOTE` reads as
when two are), and `remote` maps each facet value that `_REMOTE_FACET` names as
remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A
tenant with neither states no such dimension, and the runner then does
not filter on it."""
@@ -184,7 +181,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
country = found
for v in values:
word = " ".join(str(v.get("descriptor") or "").split())
if v.get("id") and _remote(word) == "remote":
if v.get("id") and word.lower() in _REMOTE_FACET:
remote[word.lower()] = f"{name}={v['id']}"
out: dict[str, dict[str, str]] = {}
if country:
@@ -216,11 +213,14 @@ def _today() -> datetime:
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and
#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a
#: word not listed, a bare "No" among them, says nothing and stays unknown.
_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
#: The `remoteType` words a facet names that mean remote, which the `remote`
#: filter looks for; the row keeps the host's word whatever it is (only some
#: tenants send it, each in its own way: Zendesk's "Fully Flexible", Axis'
#: "On-Site").
_REMOTE_FACET = ("remote", "fully flexible")
#: The words that put a role's places in an office, for their `remote` flag.
_OFFICE = ("hybrid", "in office", "on-site", "onsite")
def _requisition(posting: dict[str, Any], path: str) -> str:
@@ -234,12 +234,10 @@ def _requisition(posting: dict[str, Any], path: str) -> str:
def _remote(value: Any) -> str | None:
return _REMOTE.get(str(value or "").strip().lower())
"""The host's `remoteType`, as given."""
return str(value or "").strip() or None
#: `timeType` -> the app's employment words.
_EMPLOYMENT = {"full time": "full_time", "part time": "part_time"}
#: What `locationsText` says of a role at several places: "2 Locations". The
#: places arrive with the role's own read.
_COUNT = re.compile(r"\d+ locations?", re.I)
@@ -259,8 +257,9 @@ def _place(text: Any, remote: str | None) -> Place | None:
if len(parts) > 1 and "," not in text and all(parts) \
and not parts[0].lower().startswith(("remote", "hybrid", "on-site", "onsite")):
text = ", ".join(reversed(parts))
return {"text": text, "remote": {"remote": True, "hybrid": False, "onsite": False}.get(
remote or "")}
word = (remote or "").lower()
return {"text": text, "remote": True if word in _REMOTE_FACET
else False if word in _OFFICE else None}
def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
@@ -318,9 +317,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if places := _places(posting, remote):
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
if len(places) == 1 and (code := _country_code(posting)):
out["country"] = code
if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
if employment := str(posting.get("timeType") or "").strip():
out["employment"] = employment
if posted := parse_dt(posting.get("startDate")):
out["posted"] = posted
@@ -329,15 +326,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
return out
def _country_code(posting: dict[str, Any]) -> str | None:
"""The role's country as an ISO code, from the requisition's location. The
code belongs to the main location only, so it is used for a role with one
place."""
country = (posting.get("jobRequisitionLocation") or {}).get("country") or {}
code = str(country.get("alpha2Code") or "").strip().upper()
return code if len(code) == 2 else None
def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
"""The main location and the additional ones, each once, which replace the
listing's "2 Locations". A single location that does not name its country
+16
View File
@@ -0,0 +1,16 @@
"""`upsert`: a role stored as a pull stores it and then mapped, as the prep
stage after the pull maps it. For tests of what the feed, the gates and the
pages read, which are prep's values, not the raw ones `upsert_role` keeps."""
from __future__ import annotations
from typing import Any
from app import prep
from app.storage import Store
def upsert(store: Store, item: Any, subscription: Any = None) -> bool:
new = store.upsert_role(item, subscription)
prep.run(store, dupes=False)
return new
+1 -1
View File
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
#: no database, so storage may use them. `main` assembles the app from
#: everything below it; `cli` and `__main__` are the entry points and may
#: reach anything.
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli",
"__main__"]
APP = "app"
EXTERNAL = {"desk", "pipeline", "auth"}
+7 -6
View File
@@ -132,10 +132,10 @@ def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
assert rows[0]["location"] == "France"
assert rows[0]["remote"] is True
assert rows[0]["remote"] == "remote"
assert rows[0]["department"] == "Product"
assert rows[1]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is False
assert rows[1]["remote"] == "hybrid"
assert rows[0]["description"] == "" # the list endpoint carries no text
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
@@ -173,8 +173,9 @@ def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
assert rows[0]["description"] == "Grow our pipeline."
assert rows[0]["remote"] is True # "Remote - Germany"
assert rows[1]["remote"] is None # "Munich"
assert rows[0]["places"][0]["remote"] is True # "Remote - Germany"
assert rows[1]["places"][0]["remote"] is None # "Munich"
assert "remote" not in rows[0] # the feed states none
def test_personio_reads_salary_information_when_present() -> None:
@@ -291,9 +292,9 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
assert country_of(rows[0]["location"]) == "SE"
def test_teamtailor_maps_remote_status_and_falls_back_to_role_for_department() -> None:
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
assert [r["remote"] for r in rows] == ["hybrid", "remote", None] # none says nothing
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
assert rows[1]["title"] == "Läkare" # trimmed
assert rows[1]["department"] == "Software Development"
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
+4 -4
View File
@@ -25,14 +25,14 @@ def test_the_primary_and_every_secondary_location_are_places() -> None:
assert second["places"] == [{"text": "Dublin", "remote": True}]
def test_a_hybrid_role_is_not_remote_although_the_host_flags_it() -> None:
def test_remote_is_the_workplace_type_as_the_host_writes_it() -> None:
first, second = ashby.parse(BOARD, "acme")
assert first["remote"] == "hybrid" and second["remote"] == "remote"
assert first["remote"] == "Hybrid" and second["remote"] == "Remote"
def test_the_employment_type_is_mapped() -> None:
def test_the_employment_type_is_the_hosts_word() -> None:
first, second = ashby.parse(BOARD, "acme")
assert (first["employment"], second["employment"]) == ("full_time", "contract")
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
+7 -7
View File
@@ -47,15 +47,15 @@ def test_a_listing_becomes_rows() -> None:
assert second["url"] == "https://configit.bamboohr.com/careers/37"
assert second["department"] == "Marketing" and rows[0]["department"] == ""
assert second["location"] == "København Ø"
assert second["remote"] == "hybrid"
assert second["remote"] == "2"
assert (second["posted"], second["description"], second["salary_text"]) == (None, "", "")
@pytest.mark.parametrize(("flag", "kind", "remote"), [
(True, "0", "remote"), (None, "0", "onsite"), (None, "1", "remote"), (None, "2", "hybrid"),
(None, None, None), (False, "7", None)])
(True, "0", True), (None, "0", "0"), (None, "1", "1"), (None, "2", "2"),
(None, None, None), (False, "7", "7")])
def test_remote_is_the_flag_then_the_location_type(
flag: bool | None, kind: str | None, remote: str | None) -> None:
flag: bool | None, kind: str | None, remote: bool | str | None) -> None:
body = json.loads(_read("bamboohr.json"))
body["result"][0] |= {"isRemote": flag, "locationType": kind}
assert bamboohr.parse(json.dumps(body).encode(), "configit")[0]["remote"] == remote
@@ -102,7 +102,7 @@ def test_detail_reads_the_text_the_date_and_the_country() -> None:
assert found["posted"] == datetime(2026, 9, 23, tzinfo=UTC)
assert found["location"] == "København Ø, Denmark"
assert found["places"] == [{"text": "København Ø, Denmark", "remote": None}]
assert found["remote"] == "hybrid"
assert found["remote"] == "2"
text = found["description"]
assert text.startswith("Our Global Marketing Team is looking for")
assert "Skills & Experience" in text and "&amp;" not in text and "<" not in text
@@ -144,11 +144,11 @@ def test_the_board_and_its_details_are_read_with_the_json_accept_on_the_detail()
def test_the_role_states_one_place_and_the_employment() -> None:
rows = bamboohr.parse(_read("bamboohr.json"), "configit")
assert rows[1]["places"] == [{"text": "København Ø", "remote": None}]
assert rows[1]["employment"] == "full_time"
assert rows[1]["employment"] == "Full-Time"
body = json.loads(_read("bamboohr.json"))
body["result"][0] |= {"employmentStatusLabel": "Seasonal", "location": {"city": None}}
first = bamboohr.parse(json.dumps(body).encode(), "configit")[0]
assert first["places"] == [] and "employment" not in first
assert first["places"] == [] and first["employment"] == "Seasonal"
def test_it_declares_what_it_reads() -> None:
+5 -5
View File
@@ -32,7 +32,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
assert first["url"] == ("https://jobs.deel.com/klarna/job-details/"
"21584046-56d5-44c4-8b18-7e15dc659827/overview")
assert first["location"] == "Stockholm" and first["department"] == "Engineering"
assert first["remote"] == "onsite"
assert first["remote"] == "ON_SITE"
assert first["description"].startswith("Klarna, briefly At Klarna")
assert "<" not in first["description"] and "$" not in first["description"][:3]
assert all(r["description"] for r in rows)
@@ -41,18 +41,18 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
"2026-09-07T12:33:38.484000+00:00"
def test_several_locations_are_joined_and_remote_uses_the_stores_words() -> None:
def test_several_locations_are_joined_and_remote_is_the_hosts_word() -> None:
rows = {r["source_id"][:8]: r for r in deel.parse(PAGE, "klarna")}
assert rows["a8be45ac"]["location"] == "Stockholm; London; Milan; Warsaw"
assert rows["f96bb9d4"]["remote"] == "remote"
assert rows["f96bb9d4"]["remote"] == "REMOTE"
assert rows["f96bb9d4"]["location"] == "Ohio"
# Every place is kept, in the host's order; the work arrangement is the
# places' remote flag.
assert rows["a8be45ac"]["places"] == [
{"text": name, "remote": False} for name in ("Stockholm", "London", "Milan", "Warsaw")]
assert rows["f96bb9d4"]["places"] == [{"text": "Ohio", "remote": True}]
assert rows["f96bb9d4"]["employment"] == "contract" # "Consultant"
assert rows["21584046"]["employment"] == "full_time"
assert rows["f96bb9d4"]["employment"] == "Consultant"
assert rows["21584046"]["employment"] == "Full-time"
def test_pay_is_only_text_since_the_period_is_not_stated() -> None:
+2 -2
View File
@@ -87,8 +87,8 @@ def test_a_page_becomes_rows() -> None:
@pytest.mark.parametrize(("option", "remote"), [
("onsite", "onsite"), ("hybrid", "hybrid"), ("remote", "remote"), (None, None), ("odd", None)])
def test_the_work_location_option_maps_to_the_remote_words(
("onsite", "onsite"), ("Hybrid", "Hybrid"), (None, None), ("odd", "odd"), (" ", None)])
def test_the_work_location_option_is_the_hosts_word(
option: str | None, remote: str | None) -> None:
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
+6 -6
View File
@@ -55,7 +55,7 @@ def test_a_listing_becomes_rows() -> None:
assert first["source_id"] == "8080046002" # numeric, from a synced Greenhouse
assert first["title"] == "Staff AI Product Engineer (Prague/Brno/Remote EU)"
assert first["url"] == "https://jobs.gem.com/productboard/8080046002"
assert first["department"] == "Product Engineering" and first["remote"] == "hybrid"
assert first["department"] == "Product Engineering" and first["remote"] == "HYBRID"
assert first["location"] == "Prague"
assert (first["posted"], first["description"], first["salary_text"]) == (None, "", "")
assert backend["source_id"] == "am9icG9zdDqWC1V_RM7U6ORvTd1Iu64r" # opaque
@@ -64,16 +64,16 @@ def test_a_listing_becomes_rows() -> None:
assert backend["places"] == [{"text": "Prague", "remote": False},
{"text": "Remote Europe", "remote": True},
{"text": "Brno", "remote": False}]
assert first["employment"] == "full_time"
assert first["employment"] == "FULL_TIME"
assert len({r["source_id"] for r in rows}) == 11
@pytest.mark.parametrize(("kind", "flags", "remote"), [
("IN_OFFICE", [False], "onsite"), ("HYBRID", [True], "hybrid"), ("REMOTE", [False], "remote"),
(None, [True, True], "remote"), (None, [True, False], None), (None, [], None),
("SOMETHING_NEW", [False], None)])
("IN_OFFICE", [False], "IN_OFFICE"), ("HYBRID", [True], "HYBRID"), ("REMOTE", [False], "REMOTE"),
(None, [True, True], True), (None, [True, False], None), (None, [], None),
("SOMETHING_NEW", [False], "SOMETHING_NEW")])
def test_remote_is_the_location_type_then_the_locations(
kind: str | None, flags: list[bool], remote: str | None) -> None:
kind: str | None, flags: list[bool], remote: bool | str | None) -> None:
posting = {"extId": "1", "title": "Role", "job": {"locationType": kind},
"locations": [{"city": "X", "isRemote": f} for f in flags]}
assert gem.rows([posting], "acme")[0]["remote"] == remote
+4 -4
View File
@@ -51,7 +51,7 @@ def test_a_board_of_general_applications_reads_as_it_is() -> None:
rows = hibob.parse(DIXA, "dixa")
assert [(r["title"], r["location"], r["remote"]) for r in rows] == [
("Expression of interest", "Copenhagen, Denmark", "hybrid"),
("Expression of interest", "London, United Kingdom", "onsite")]
("Expression of interest", "London, United Kingdom", "on_site")]
assert rows[1]["description"].startswith("We're always excited")
@@ -153,10 +153,10 @@ def test_each_ad_names_one_site_and_the_host_lists_a_copy_per_site() -> None:
assert len({r["source_id"] for r in dixa}) == 2
def test_the_employment_type_is_mapped_and_a_permanent_one_is_left_unmapped() -> None:
def test_the_employment_type_is_the_hosts_word() -> None:
rows = hibob.parse(BOARD, "liquidia")
assert rows[0]["employment"] == "full_time" # "Regular Full Time"
assert "employment" not in hibob.parse(DIXA, "dixa")[0] # "Permanent"
assert rows[0]["employment"] == "Regular Full Time"
assert hibob.parse(DIXA, "dixa")[0]["employment"] == "Permanent"
def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
+1 -1
View File
@@ -57,7 +57,7 @@ def test_a_page_becomes_rows() -> None:
"Manchester, United Kingdom"]
assert "country" not in several
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
assert (first["country"], first["employment"]) == ("RO", "full_time")
assert (first["country"], first["employment"]) == ("RO", "FULL_TIME")
assert all(r["description"] and "<" not in r["description"] and "&nbsp;" not in r["description"]
for r in rows)
+9 -14
View File
@@ -14,7 +14,6 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models as app_models # noqa: E402
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jobylon # noqa: E402
@@ -36,20 +35,20 @@ def test_rows_come_from_the_embeds_literal() -> None:
assert first["location"] == "Stockholm"
assert first["places"] == [{"text": "Stockholm", "remote": None}]
assert first["department"] == "Accounting & Finance" # `&` read as `&`
assert first["remote"] == "hybrid"
assert first["remote"] == "Hybrid"
assert first["description"] == "" and first["salary_text"] == ""
# A day, with no time of day: the embed says "Sept. 25, 2026".
assert first["posted"] is not None and first["posted"].isoformat() == \
"2026-09-25T00:00:00+00:00"
def test_places_department_and_remote_follow_the_embeds_words() -> None:
def test_places_department_and_remote_are_the_embeds_words() -> None:
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
assert rows["384238"]["location"] == "Houston; Dallas; Atlanta"
assert [p["text"] for p in rows["384238"]["places"]] == ["Houston", "Dallas", "Atlanta"]
assert rows["384238"]["department"] == "" # "All functions" names none
assert rows["382124"]["remote"] == "remote"
assert rows["382860"]["remote"] == "onsite"
assert rows["382124"]["remote"] == "Remote"
assert rows["382860"]["remote"] == "On-site"
assert rows["380347"]["remote"] is None # a role with no workspace
@@ -130,18 +129,14 @@ def test_an_empty_board_is_no_roles() -> None:
assert jobylon.parse(EMPTY, "1056") == []
def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged(
caplog: pytest.LogCaptureFixture) -> None:
caplog.set_level("DEBUG", logger="plugins.readers.jobylon")
def test_the_experience_and_the_employment_type_are_the_hosts_words() -> None:
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
assert rows["384238"]["seniority"] == "mid" # "Mid level"
assert "seniority" not in rows["385191"] # "Experienced" says too little
assert "unknown experience 'Experienced'" in caplog.text
assert rows["385191"]["employment"] == "full_time"
assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values())
assert rows["384238"]["seniority"] == "Mid level"
assert rows["385191"]["seniority"] == "Experienced"
assert rows["385191"]["employment"] == "Full-time"
rows = {r["source_id"]: r for r in jobylon.parse(
(FIXTURES / "jobylon-places.html").read_bytes(), "3143")}
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship"
assert rows["502"]["seniority"] == "Student" and rows["502"]["employment"] == "Internship"
def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None:
+2 -2
View File
@@ -28,9 +28,9 @@ def test_the_country_is_only_taken_for_a_role_at_one_place() -> None:
assert "country" not in several and one["country"] == "DE"
def test_the_commitment_is_mapped_and_a_permanent_one_is_left_unmapped() -> None:
def test_the_commitment_is_the_hosts_word() -> None:
several, one = lever.parse(BOARD, "acme")
assert several["employment"] == "full_time" and "employment" not in one
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
+5 -5
View File
@@ -59,19 +59,19 @@ def test_the_locations_are_the_primary_and_the_secondary_ones() -> None:
assert rows[2]["location"] == "India"
def test_every_place_is_kept_with_its_work_mode_and_the_countrys_code() -> None:
def test_every_place_is_kept_with_its_work_mode() -> None:
rows = _rows()
assert [p["text"] for p in rows[1]["places"]] == [
"Malaysia", "Philippines", "Indonesia", "Thailand"]
assert [p["remote"] for p in rows[0]["places"]] == [False, False] # hybrid
assert [p["remote"] for p in rows[4]["places"]] == [True] # remote
assert [p["remote"] for p in rows[3]["places"]] == [None] # states none
# One country when the places share one, none over several.
assert [r["country"] for r in rows] == [None, None, "IN", "DE", "US"]
assert all("country" not in r for r in rows) # prep derives it from the places
def test_remote_follows_the_workplace_type_code() -> None:
assert [r["remote"] for r in _rows()] == ["hybrid", "hybrid", "onsite", None, "remote"]
def test_remote_is_the_workplace_type_code_as_given() -> None:
assert [r["remote"] for r in _rows()] == [
"ORA_HYBRID", "ORA_HYBRID", "ORA_ON_SITE", None, "ORA_REMOTE"]
def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
+5 -19
View File
@@ -3,17 +3,14 @@ No network: the fixtures are saved feeds."""
from __future__ import annotations
import logging
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.models import SENIORITIES # noqa: E402
from plugins.readers import Ask, personio # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -37,23 +34,12 @@ def test_additional_offices_are_places_of_the_role_each_once() -> None:
assert [p["text"] for p in first["places"]] == ["Remote - Germany", "Berlin", "Hamburg"]
assert [p["remote"] for p in first["places"]] == [True, None, None]
assert first["location"] == "Remote - Germany; Berlin; Hamburg"
assert first["remote"] is True # one of the places is remote
assert "remote" not in first # the feed states none
assert _rows()[1]["places"] == [{"text": "Munich", "remote": None}]
assert _rows()[1]["remote"] is None
def test_the_stated_level_and_employment_are_mapped() -> None:
def test_the_stated_level_and_employment_are_the_hosts_words() -> None:
rows = _rows()
assert [r["seniority"] for r in rows] == ["junior", "exec", "intern"]
# permanent + full-time; permanent + full-time; working student wins over part-time
assert [r["employment"] for r in rows] == ["full_time", "full_time", "internship"]
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
caplog: pytest.LogCaptureFixture) -> None:
assert {v for v in personio._SENIORITY.values() if v} <= set(SENIORITIES)
with caplog.at_level(logging.DEBUG, logger=personio.__name__):
assert personio._seniority("wizard") is None
assert personio._seniority("experienced") is None # known, names no level
assert personio._seniority("") is None
assert [r.getMessage() for r in caplog.records] == ["personio: unknown seniority 'wizard'"]
assert [r["seniority"] for r in rows] == ["entry-level", "executive", "student"]
# permanent + full-time gives the schedule; a working student is the contract's word
assert [r["employment"] for r in rows] == ["full-time", "full-time", "working_student"]
+1 -1
View File
@@ -226,7 +226,7 @@ def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() ->
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
assert got["description"].startswith("Craft: Games Design & Production Job Description:")
assert "<" not in got["description"] and "&amp;" not in got["description"]
assert got["remote"] == "onsite"
assert got["remote"] == "On-site"
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
+2 -5
View File
@@ -161,9 +161,6 @@ def test_a_role_has_its_one_place_with_the_work_mode_of_the_role() -> None:
assert rows[0]["location"] == "Bucharest"
def test_employment_is_read_from_the_type_and_an_unknown_one_is_left() -> None:
def test_employment_is_the_hosts_type_as_given() -> None:
rows = pinpoint.parse((FIXTURES / "pinpoint.json").read_bytes(), "xeneta")
assert [r["employment"] for r in rows] == ["full_time", "temporary"]
assert pinpoint._employment("permanent_part_time") == "part_time"
assert pinpoint._employment("internship") == "internship"
assert pinpoint._employment("something_new") is None
assert [r["employment"] for r in rows] == ["permanent_full_time", "fixed_term_full_time"]
+9 -25
View File
@@ -4,17 +4,14 @@ bunq's board, read on 2026-09-30."""
from __future__ import annotations
import logging
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.models import SENIORITIES # noqa: E402
from plugins.readers import Ask, recruitee # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -27,19 +24,18 @@ def _rows() -> list[dict]:
def test_the_declaration_says_what_the_feed_gives() -> None:
reads = recruitee.READS
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
assert reads.robots.startswith("api: no robots.txt check needed")
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
def test_every_location_is_a_place_and_the_country_is_stated_when_they_share_one() -> None:
def test_every_location_is_a_place_and_no_country_is_stated() -> None:
first, _remote, berlin = _rows()
assert [p["text"] for p in first["places"]] == [
"Amsterdam, Noord-Holland, Netherlands", "Sofia, Bulgaria"]
assert first["location"] == "Amsterdam, Noord-Holland, Netherlands; Sofia, Bulgaria"
assert first["country"] is None # NL and BG
assert "country" not in first # prep derives it from the places
assert berlin["places"] == [{"text": "Berlin, Germany", "remote": None}]
assert berlin["country"] == "DE"
def test_remote_job_is_not_a_place() -> None:
@@ -48,28 +44,16 @@ def test_remote_job_is_not_a_place() -> None:
assert remote["location"] == "Remote"
offer = {"id": 1, "title": "T", "careers_url": "https://a.example/o/t", "location": "Remote job",
"remote": False}
assert recruitee._places(offer) == ([], set())
assert recruitee._places(offer) == []
def test_an_offer_without_a_locations_list_falls_back_to_its_location() -> None:
offer = {"location": "Lisbon, Portugal", "country_code": "pt"}
assert recruitee._places(offer) == ([{"text": "Lisbon, Portugal", "remote": None}], {"PT"})
assert recruitee._places(offer) == [{"text": "Lisbon, Portugal", "remote": None}]
def test_the_stated_level_and_employment_are_mapped() -> None:
def test_the_stated_level_and_employment_are_the_hosts_codes() -> None:
rows = _rows()
assert [r["seniority"] for r in rows] == [None, "junior", "intern"]
assert [r["employment"] for r in rows] == ["full_time", "part_time", "internship"]
assert recruitee._employment("fulltime_fixed_term") == "temporary"
assert recruitee._employment("freelance") == "contract"
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
caplog: pytest.LogCaptureFixture) -> None:
assert {v for v in recruitee._SENIORITY.values() if v} <= set(SENIORITIES)
with caplog.at_level(logging.DEBUG, logger=recruitee.__name__):
assert recruitee._seniority("senior_level") is None
assert recruitee._employment("odd_job") is None
assert [r.getMessage() for r in caplog.records] == [
"recruitee: unknown experience_code 'senior_level'",
"recruitee: unknown employment_type_code 'odd_job'"]
assert [r["seniority"] for r in rows] == ["experienced", "entry_level", "student_school"]
assert [r["employment"] for r in rows] == [
"fulltime_permanent", "parttime_permanent", "internship"]
+10 -14
View File
@@ -41,21 +41,18 @@ def test_rows_merge_one_entry_per_place_into_one_role() -> None:
assert first["description"] == "" and first["salary_text"] == "" and first["posted"] is None
def test_remote_is_true_only_when_every_place_is_a_remote_one() -> None:
def test_a_remote_place_is_flagged_and_the_role_states_no_remote() -> None:
by_id = {r["source_id"][:8]: r for r in rippling.parse(BOARD, "rippling")}
assert by_id["46b4af52"]["remote"] is True and by_id["19ae5b34"]["remote"] is True
assert by_id["75ad50c6"]["remote"] is None # an office label says nothing either way
mixed = json.dumps([
{"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Remote (Germany)"}},
{"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Berlin"}}]).encode()
assert rippling.parse(mixed, "x")[0]["remote"] is None
assert [p["remote"] for p in by_id["46b4af52"]["places"]] == [True]
assert [p["remote"] for p in by_id["75ad50c6"]["places"]] == [None] # an office label
assert all("remote" not in r for r in by_id.values()) # prep derives it from the places
def test_prisma_board_reads_two_roles() -> None:
rows = rippling.parse(PRISMA, "prisma-careers")
assert [(r["title"], r["department"], r["location"], r["remote"]) for r in rows] == [
("General Applications: Engineering", "Engineering", "Remote (Germany)", True),
("General Applications: Product", "Product", "Remote (Germany)", True)]
assert [(r["title"], r["department"], r["location"]) for r in rows] == [
("General Applications: Engineering", "Engineering", "Remote (Germany)"),
("General Applications: Product", "Product", "Remote (Germany)")]
def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None:
@@ -92,7 +89,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
assert "<" not in found["description"]
assert found["location"] == "San Francisco, CA; Seattle, WA; New York, NY"
assert [p["text"] for p in found["places"]] == ["San Francisco, CA", "Seattle, WA", "New York, NY"]
assert found["employment"] == "full_time" # employmentType.label SALARIED_FT
assert found["employment"] == "SALARIED_FT" # employmentType.label
assert found["posted"] is not None and found["posted"].isoformat() == \
"2025-04-07T12:29:19.348000-07:00"
assert (found["salary_min"], found["salary_max"], found["currency"]) == (216000, 378000, "USD")
@@ -203,13 +200,12 @@ def test_discovery_finds_the_board_slug() -> None:
("rippling", "prisma-careers"), ("rippling", "Acme"), ("rippling", "beta")]
def test_a_role_at_several_places_keeps_each_once_and_is_remote_only_if_all_are() -> None:
def test_a_role_at_several_places_keeps_each_once() -> None:
rows = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}
director = rows[DIRECTOR]
assert [p["text"] for p in director["places"]] == [
"San Francisco, CA", "Seattle, WA", "New York, NY"]
assert director["location"] == "San Francisco, CA; Seattle, WA; New York, NY"
assert director["remote"] is None
remote = next(r for r in rows.values() if r["places"] == [
{"text": "Remote (United States)", "remote": True}])
assert remote["remote"] is True
assert remote["places"][0]["remote"] is True
+5 -23
View File
@@ -4,17 +4,14 @@ No network: the shapes are those of Bosch's board, read on 2026-09-30."""
from __future__ import annotations
import json
import logging
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.models import SENIORITIES # noqa: E402
from plugins.readers import Ask, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -54,38 +51,23 @@ def test_page_reads_by_offset_and_takes_the_ask() -> None:
def test_the_place_is_one_clean_line_with_its_remote_flag_and_country() -> None:
[row] = _rows(_posting())
assert row["places"] == [{"text": "Hatvan, Hungary", "remote": False}] # ", ," gone
assert row["location"] == "Hatvan, Hungary" and row["country"] == "HU"
assert row["location"] == "Hatvan, Hungary" and row["country"] == "hu"
[remote] = _rows(_posting(location={"remote": True, "fullLocation": "Remote", "country": ""}))
assert remote["places"] == [{"text": "Remote", "remote": True}] and remote["country"] is None
[none] = _rows(_posting(location={}))
assert none["places"] == [] and none["location"] == ""
def test_the_stated_level_and_employment_are_mapped() -> None:
def test_the_stated_level_and_employment_are_the_hosts_words() -> None:
[row] = _rows(_posting(experienceLevel={"id": "internship", "label": "Internship"},
typeOfEmployment={"id": "intern", "label": "Intern"}))
assert (row["seniority"], row["employment"]) == ("intern", "internship")
levels = {"entry_level": "junior", "director": "director", "executive": "exec",
"associate": None, "mid_senior_level": None, "not_applicable": None}
for word, want in levels.items():
[row] = _rows(_posting(experienceLevel={"id": word}))
assert row["seniority"] == want, word
[row] = _rows(_posting(typeOfEmployment={"id": "permanent"}))
assert row["employment"] is None
assert (row["seniority"], row["employment"]) == ("internship", "intern")
[row] = _rows(_posting(experienceLevel={"id": "mid_senior_level"}))
assert row["seniority"] == "mid_senior_level"
[row] = _rows(_posting())
assert row["seniority"] is None and row["employment"] is None
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
caplog: pytest.LogCaptureFixture) -> None:
assert {v for v in smartrecruiters._SENIORITY.values() if v} <= set(SENIORITIES)
with caplog.at_level(logging.DEBUG, logger=smartrecruiters.__name__):
[row] = _rows(_posting(experienceLevel={"id": "guru"}))
assert row["seniority"] is None
assert [r.getMessage() for r in caplog.records] == [
"smartrecruiters: unknown experienceLevel 'guru'"]
def test_the_saved_page_still_reads() -> None:
data = json.loads((FIXTURES / "smartrecruiters.json").read_bytes())
assert [r["location"] for r in smartrecruiters.rows(data, "Acme")] == ["Berlin, Germany",
+3 -13
View File
@@ -315,7 +315,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/"
return successfactors.detail(get, "careers.vestas.com", {"url": url}) # type: ignore[arg-type]
def test_detail_reads_the_description_the_date_and_the_country_code_of_an_address() -> None:
def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None:
got = _detail("successfactors-job.html")
assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you")
assert "<" not in got["description"]
@@ -324,7 +324,7 @@ def test_detail_reads_the_description_the_date_and_the_country_code_of_an_addres
# The locality and the country's name; the region is cut to "Seou" by the site.
assert got["location"] == "Seoul, South Korea"
assert got["places"] == [{"text": "Seoul, South Korea", "remote": None}]
assert got["country"] == "KR"
assert "country" not in got # prep derives it from the places
def test_detail_keeps_every_address_of_a_role_at_several_places() -> None:
@@ -333,15 +333,6 @@ def test_detail_keeps_every_address_of_a_role_at_several_places() -> None:
assert [p["text"] for p in got["places"]] == [
"Seoul, South Korea", "Busan, South Korea", "Aarhus, Denmark"]
assert got["location"] == "Seoul, South Korea; Busan, South Korea; Aarhus, Denmark"
assert "country" not in got # KR and DK: no single country for the role
def test_detail_states_the_country_when_every_address_is_in_it() -> None:
text = _read("successfactors-places-detail.html").replace(
'content="Aarhus, DK, 8200"', 'content="Daegu, KR, 41200"')
got = successfactors.detail(lambda url, **kw: text.encode(), "careers.vestas.com",
{"url": "https://careers.vestas.com/job/x/1/"})
assert got["country"] == "KR"
def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> None:
@@ -351,11 +342,10 @@ def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> N
for r in rows)
def test_detail_reads_the_country_from_a_street_address() -> None:
def test_detail_reads_the_place_from_a_street_address() -> None:
"""Systematic and most other sites give one `streetAddress`, "City, CC, zip"."""
got = _detail("successfactors-job-street.html")
assert got["location"] == "Bern, Switzerland"
assert got["country"] == "CH"
assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC)
assert got["description"].startswith("Senior Solution Engineer Defence")
+3 -11
View File
@@ -5,11 +5,9 @@ No network: the fixture has the shape of Hugging Face's board, read on
from __future__ import annotations
import json
import logging
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
@@ -49,21 +47,15 @@ def test_every_location_of_a_role_is_a_place_once() -> None:
assert [p["text"] for p in first["places"]] == ["Berlin, Germany", "Munich, Bavaria, Germany",
"Paris, France"]
assert first["location"] == "Berlin, Germany; Munich, Bavaria, Germany; Paris, France"
assert first["country"] is None # DE and FR
assert "country" not in first # prep derives it from the places
assert [p["remote"] for p in first["places"]] == [False] * 3
assert remote["places"] == [{"text": "France", "remote": True}]
assert remote["country"] == "FR"
# A list-less answer (the older shape) still gives its one place.
assert oslo["places"] == [{"text": "Oslo, Norway", "remote": False}]
assert oslo["country"] == "NO"
def test_the_stated_hours_are_mapped_and_an_unknown_type_is_logged(
caplog: pytest.LogCaptureFixture) -> None:
with caplog.at_level(logging.DEBUG, logger=workable.__name__):
rows = _rows()
assert [r["employment"] for r in rows] == ["full_time", "part_time", None]
assert [r.getMessage() for r in caplog.records] == ["workable: unknown type 'volunteer'"]
def test_the_stated_hours_are_the_hosts_word() -> None:
assert [r["employment"] for r in _rows()] == ["full", "part", "volunteer"]
def test_the_shape_of_a_real_list_answer_is_read() -> None:
+10 -11
View File
@@ -54,8 +54,8 @@ def test_rows_read_the_listing_of_a_tenant_that_sends_remote_type() -> None:
"Associate-Solutions-Architect_R35358")
assert first["posted"] == TODAY
assert first["description"] == "" and first["department"] == "" and first["salary_text"] == ""
# "Fully Flexible" is Zendesk's word for remote.
assert [r["remote"] for r in rows] == ["remote", "onsite", "hybrid", "remote", "hybrid"]
# The host's own words, "Fully Flexible" among them; prep maps them.
assert [r["remote"] for r in rows] == ["Fully Flexible", "In Office", "Hybrid", "Fully Flexible", "Hybrid"]
def test_rows_of_a_tenant_without_remote_type_leave_remote_unknown() -> None:
@@ -80,7 +80,7 @@ def test_rows_of_a_site_called_jobs_state_the_requisition_id_and_no_date_past_30
assert rows[1]["places"] == [{"text": "Remote US TX", "remote": True}]
assert rows[2]["url"].startswith("https://redhat.wd5.myworkdayjobs.com/jobs/job/")
assert rows[3]["posted"] is None # "Posted 30+ Days Ago" says only that it is older
assert [r["remote"] for r in rows] == ["onsite", "remote", "remote", "hybrid"]
assert [r["remote"] for r in rows] == ["Onsite", "Remote", "Remote", "Hybrid"]
@pytest.mark.parametrize(("text", "expected"), [
@@ -275,13 +275,12 @@ def test_detail_reads_the_description_date_remote_and_place_of_the_role() -> Non
SLUG, row)
assert asked == [f"https://{HOST}/wday/cxs/zendesk/zendesk"
"/job/Remote-Mexico-City-Mexico/Associate-Solutions-Architect_R35358"]
assert set(got) == {"description", "location", "places", "posted", "remote", "country",
"employment"}
assert set(got) == {"description", "location", "places", "posted", "remote", "employment"}
assert got["posted"] == datetime(2026, 9, 29, tzinfo=UTC)
assert got["remote"] == "remote"
assert got["remote"] == "Fully Flexible"
assert got["location"] == "Remote, Mexico City, Mexico"
assert got["places"] == [{"text": "Remote, Mexico City, Mexico", "remote": True}]
assert (got["country"], got["employment"]) == ("MX", "full_time")
assert got["employment"] == "Full time" and "country" not in got
assert got["description"].startswith("Job Description") or len(got["description"]) > 200
assert "<" not in got["description"] and "&nbsp;" not in got["description"]
assert " " not in got["description"]
@@ -296,7 +295,7 @@ def test_detail_names_every_location_of_a_role_listed_as_several() -> None:
"/job/Remote-Italy/Telco-Cloud-Consultant_R-059566-2"]
assert got["location"] == "Remote Italy; Remote Spain"
assert [p["text"] for p in got["places"]] == ["Remote Italy", "Remote Spain"]
assert "country" not in got # the code belongs to the main location alone
assert "country" not in got
def _detail(**info: Any) -> bytes:
@@ -337,9 +336,9 @@ def test_a_country_dash_city_place_is_turned_round() -> None:
assert workday._place("12 Locations", None) is None
def test_detail_takes_no_word_for_remote_it_does_not_know() -> None:
def test_detail_hands_on_a_word_for_remote_it_does_not_know() -> None:
row = workday.rows(_load("workday.json"), SLUG)[0]
assert "remote" not in workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row)
assert workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row)["remote"] == "No"
def test_detail_refuses_a_row_that_is_not_a_role_of_the_board() -> None:
@@ -383,7 +382,7 @@ def test_detail_is_read_through_the_getter_ats_binds(
monkeypatch.setattr(ats, "_get", fake_get)
row = workday.rows(_load("workday.json"), SLUG)[0]
got = workday.detail(ats._getter(object(), "workday", SLUG), SLUG, row) # type: ignore[arg-type]
assert got["remote"] == "remote"
assert got["remote"] == "Fully Flexible"
assert seen[0][:2] == ("workday", SLUG)
+12 -11
View File
@@ -13,6 +13,7 @@ from app.models import company_fold
from app.storage import Store
from app.storage.schema import _add_company_aliases
from desk.models import Item
from prepped import upsert
def undo_v20(conn: sqlite3.Connection) -> None:
@@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None:
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "ORSTED A/S"))
store.upsert_role(posting("3", "Ørsted"))
store.upsert_role(posting("4", "Nobody"))
upsert(store, posting("1", "Orsted"))
upsert(store, posting("2", "ORSTED A/S"))
upsert(store, posting("3", "Ørsted"))
upsert(store, posting("4", "Nobody"))
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
assert [c.key for c in store.companies()] == ["ørsted"]
@@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
def test_an_added_alias_finds_the_company(store: Store) -> None:
store.add_company("Ørsted")
store.add_alias("ørsted", "Dong Energy")
store.upsert_role(posting("1", "Dong Energy A/S"))
upsert(store, posting("1", "Dong Energy A/S"))
assert company_of(store, "1") == "ørsted"
with pytest.raises(KeyError):
store.add_alias("nope", "X")
@@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(store.companies()) == 3
# The company with the name's own key wins when a role names it.
store.upsert_role(posting("1", "Orsted"))
upsert(store, posting("1", "Orsted"))
assert company_of(store, "1") == "orsted"
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
store.add_company("Ørsted")
store.add_company("Orsted")
store.upsert_role(posting("1", "Orsted"))
store.upsert_role(posting("2", "Ørsted"))
upsert(store, posting("1", "Orsted"))
upsert(store, posting("2", "Ørsted"))
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
board="greenhouse")
a, b = store.save_agent("A"), store.save_agent("B")
@@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
assert store.duplicate_companies() == []
assert store.company_aliases() == {"ørsted": ["orsted"]}
# A later role from the gone spelling lands on the kept company.
store.upsert_role(posting("3", "Orsted"))
upsert(store, posting("3", "Orsted"))
assert company_of(store, "3") == "ørsted"
with pytest.raises(ValueError):
store.merge_companies("ørsted", "ørsted")
@@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21]
assert reopened.migrations_applied == [20, 21, 22]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21]
assert reopened.migrations_applied == [21, 22]
assert {c.key: c.all_roles for c in reopened.companies()} == {
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
+1 -1
View File
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21]
assert migrated.migrations_applied == [18, 19, 20, 21, 22]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+9 -7
View File
@@ -250,16 +250,16 @@ class _Context:
return t
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None:
context = _Context()
empty = Tally(results=({"filled": 0},))
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4},
context=context)
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
assert out["next"] == "extract" and context.made[0].carries["at"] == 8
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
{"cascade": "pull", "task": "enrich", "at": 4}, context=context)
assert out["next"] == "prep" and context.made[1].carries["at"] == 5
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
context=context)
@@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
assert started is not None
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
key=lambda j: j.id)
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \
["enrich", "prep", "embed", "score"]
assert [j.kind for j in jobs if j.kind != FINAL] == \
["enrich", enrich.KIND, "prep", "embed", "score"]
assert web["asked"] == [f"{BASE}/acme/1"]
embedded = next(j for j in jobs if j.kind == "embed")
assert embedded.result["embedded"] == 1
+15 -14
View File
@@ -16,6 +16,7 @@ from app.config import Config
from app.jobs import extract
from app.models import Limits
from app.storage import Store
from prepped import upsert
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
@@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
uid = posting("1").uid
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
@@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
@@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
def test_the_next_upsert_keeps_what_extract_filled(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
for r in roles])
modeltask.run(extract.TASK, config, store)
store.upsert_role(posting("1"))
upsert(store, posting("1"))
role = store.role(posting("1").uid)
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
@@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled(
def test_closed_and_answered_roles_are_not_offered(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
for sid in ("open", "closed", "done"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
ranked(store, posting("open"), posting("closed"), posting("done"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
@@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered(
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
ranked(store, posting("1"))
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
@@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
the boards produced. A role nobody ranks is never sent; a batch dealt a
uid finishes it."""
for sid in ("best", "second", "low", "unscored", "gated"):
store.upsert_role(posting(sid))
upsert(store, posting(sid))
one = store.save_agent("One", "python backend",
limits=Limits(extract_top=1, extract_min=0.5))
two = store.save_agent("Two", "python backend", enabled=False)
@@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None:
def test_disabled_finds_nothing_and_asks_nobody(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
model = stub(monkeypatch, lambda roles: [])
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
@@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
def test_a_role_read_before_without_a_level_is_read_again_for_it(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", seniority="senior")
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
@@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
withtext = posting("text", title="Senior Engineer", seniority="mid")
noText = posting("teaser", title="Analyst")
noText.body = ""
store.upsert_role(withtext)
store.upsert_role(noText)
upsert(store, withtext)
upsert(store, noText)
with store.transaction() as conn:
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
"WHERE source_id = 'teaser'")
@@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
def test_an_answer_with_no_level_is_charged_until_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1")
store.upsert_role(item)
upsert(store, item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
@@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up(
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
for item in items:
store.upsert_role(item)
upsert(store, item)
one = store.save_agent("One", "python backend")
two = store.save_agent("Two", "python backend")
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
@@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None:
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
("5", "lever", "senior", "")]:
store.upsert_role(posting(sid, board=board, seniority=stated))
upsert(store, posting(sid, board=board, seniority=stated))
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
(model, sid))
+19 -18
View File
@@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed
from app.storage import Store
from test_places import LINES, ORACLE_TOKENS
from prepped import upsert
def gazetteer() -> Gazetteer:
@@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None:
def test_a_joined_line_is_several_places(store: Store) -> None:
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
uid = posting("1").uid
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
(2, "Warsaw", "Warsaw")]
@@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None:
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
store.upsert_role(posting("1", place="Berlin, Munich", places=[
upsert(store, posting("1", place="Berlin, Munich", places=[
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
{"text": "Remote (DE)", "remote": True}]))
assert texts(store, posting("1").uid) == [
@@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
def test_places_without_a_line_make_the_line(store: Store) -> None:
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
{"text": "Malmö", "remote": None}]))
role = store.role(posting("1").uid)
assert role is not None
@@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None:
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
store.upsert_role(posting("1"))
upsert(store, posting("1", place="Aarhus"))
upsert(store, posting("1"))
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
store.upsert_role(posting("1", place="Odense"))
upsert(store, posting("1", place="Odense"))
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
store.upsert_role(posting("1", place="Aarhus"))
upsert(store, posting("1", place="Aarhus"))
uid = posting("1").uid
with sqlite3.connect(store.path) as conn:
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
store.upsert_role(posting("1", place="Aalborg"))
upsert(store, posting("1", place="Aalborg"))
assert [(p.text, p.url) for p in store.role_places(uid)] == [
("Aalborg", ""), ("Odense", "https://copy")]
def test_an_extracted_place_is_resolved(store: Store) -> None:
store.upsert_role(posting("1"))
upsert(store, posting("1"))
uid = posting("1").uid
store.set_extracted(uid, {"place": "Lund, Sweden"})
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
@@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]:
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
for key, line in lines.items():
store.upsert_role(posting(key, place=line))
upsert(store, posting(key, place=line))
return {key: posting(key).uid for key in lines}
@@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
store.upsert_role(posting("1", country="DK"))
upsert(store, posting("1", country="DK"))
assert uids(store, "country:DK") == {posting("1").uid}
@@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store,
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
store.upsert_role(posting(str(n), place=line))
upsert(store, posting(str(n), place=line))
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
("jena", "Jena", 1)]
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
def test_an_alias_links_what_it_names(store: Store) -> None:
store.upsert_role(posting("1", place="Kbh. K"))
store.upsert_role(posting("2", place="Copenhagen"))
upsert(store, posting("1", place="Kbh. K"))
upsert(store, posting("2", place="Copenhagen"))
uid = posting("1").uid
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
@@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None:
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen; Jena"))
store.upsert_role(posting("2", place="Paderborn, Germany"))
upsert(store, posting("1", place="Oberkochen; Jena"))
upsert(store, posting("2", place="Paderborn, Germany"))
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
store.add_place("Paderborn", "city", "country:DE")
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
@@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
def test_relink_places_resolves_every_role_again(store: Store) -> None:
store.upsert_role(posting("1", place="Oberkochen"))
upsert(store, posting("1", place="Oberkochen"))
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE role_places SET place_id = "
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
@@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup
lines = {}
for n, (place, country) in enumerate(LINES):
item = posting(f"line{n}", place=place, country=country)
store.upsert_role(item)
upsert(store, item)
lines[item.uid] = (place, country)
return store, lines
+378
View File
@@ -0,0 +1,378 @@
"""`app.prep`: the host's words mapped centrally, the stage over the due
roles, the unmapped list, the table fingerprint and migration 22. The host
tables here are fakes; each real table is held to its host by R1's and R2's
tests, and to the vocabularies by the last tests of this file."""
from __future__ import annotations
import importlib
import json
import sqlite3
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import cli, models, prep # noqa: E402
from app.prep import hosts, words # noqa: E402
from app.prep.country import country, of_host # noqa: E402
from app.storage import Store # noqa: E402
from desk.models import Item # noqa: E402
from plugins.readers import _shared # noqa: E402
TABLES: dict[str, Any] = {
"acme": {
"seniority": {"experienced": "", "sr": "senior"},
"employment": {"regular full time": "full_time", "permanent": ""},
"remote": {"2": "hybrid", "fully": "remote"},
},
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
"parts": {"match": {"mode": "substring",
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
}
# ------------------------------------------------------------------- words
@pytest.mark.parametrize(("field", "word", "value"), [
("seniority", "SR", "senior"),
("seniority", " Experienced ", ""), # known, names no level
("seniority", "senior", "senior"), # already the vocabulary
("seniority", "Grand Poobah", None), # unknown
("employment", "Regular Full Time", "full_time"),
("employment", "Permanent", ""),
("employment", "contract", "contract"),
("employment", "Zero hours", None),
("remote", "2", "hybrid"),
("remote", "On-site", "onsite"), # the shared words
("remote", "on_site", "onsite"),
("remote", "IN_OFFICE", "onsite"),
("remote", "office", "onsite"),
("remote", "Hybrid", "hybrid"),
("remote", "unspecified", None),
("seniority", "", ""),
])
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
field: str, word: str, value: str | None) -> None:
assert words.lookup(field, "acme", word, TABLES) == value
def test_another_hosts_table_is_not_used() -> None:
assert words.lookup("seniority", "other", "sr", TABLES) is None
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
assert words.lookup("employment", "codes", "software", TABLES) is None
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
def test_seniority_is_the_stated_level_and_the_titles() -> None:
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
assert words.remote("acme", True, (), "", TABLES) == "remote"
assert words.remote("acme", "true", (), "", TABLES) == "remote"
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
# False says nothing the places do not: hosts send it for hybrid roles too.
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
== "remote"
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
"", TABLES) == "remote"
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
== "hybrid"
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
== "remote"
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
provider="jobindex"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
prep.run(store, tables=TABLES)
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
# ----------------------------------------------------------------- country
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def ids(store: Store, *texts: str) -> list[int]:
gaz = store.gazetteer()
return [p for text in texts for p in gaz.resolve(text)]
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
gaz = store.gazetteer()
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
assert country(gaz, ids(store, "Europe")) == "EU"
assert country(gaz, [], "dk") == "DK"
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
@pytest.mark.parametrize(("stated", "code"), [
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
("", ""), ("Narnia", None)])
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
code: str | None) -> None:
assert of_host(store.gazetteer(), stated) == code
# ------------------------------------------------------------------- stage
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
**raw: Any) -> Item:
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"board": host, "company": "Acme"} | raw)
found.source = "ats"
return found
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
return found
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
places=[{"text": "Aarhus", "remote": None}]))
raw = row(store, "1")
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
("SR", "Regular Full Time", "2")
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
("", "", "", "")
assert raw["prep_due"] == 1
assert store.role_places(raw["uid"])[0].place_id is None
done = prep.run(store, tables=TABLES)
assert (done.read, done.written, done.remapped) == (1, 1, True)
mapped = row(store, "1")
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
("senior", "senior", "full_time", "hybrid", "DK", 0)
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="sr"))
prep.run(store, tables=TABLES)
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
store.upsert_role(item("2", seniority="experienced"))
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
# The place stays resolved when its text did not change.
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
assert row(store, "1")["prep_due"] == 1
assert prep.run(store, tables=TABLES).read == 2
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
store.upsert_role(item("1", seniority="sr"))
[read] = store.prep_rows("due")
store.upsert_role(item("1", seniority="experienced"))
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
assert row(store, "1")["prep_due"] == 2
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
store: Store) -> None:
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "remote")
# The host's own statement does overwrite.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "hybrid"
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
store.add_company("Ørsted")
store.upsert_role(item("1", company="ORSTED A/S"))
assert row(store, "1")["company_key"] != "ørsted"
prep.run(store, tables=TABLES)
assert row(store, "1")["company_key"] == "ørsted"
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
country="Narnia"))
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
done = prep.run(store, tables=TABLES)
assert done.unmapped == 4
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
assert listed == {("seniority", "acme", "grand poobah"): 2,
("seniority", "other", "grand poobah"): 1,
("remote", "acme", "unspecified"): 1,
("country", "acme", "narnia"): 1}
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
# A word a table learns leaves the list at the next run; the rest keep
# the moment they were first met.
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
prep.run(store, tables=learned)
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
first
assert ("seniority", "acme", "grand poobah") not in \
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
assert row(store, "1")["seniority"] == "exec"
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
store.upsert_role(item("1", seniority="Distinguished"))
store.upsert_role(item("2", seniority="Distinguished"))
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
assert prep.run(store, tables=TABLES).remapped
assert not prep.run(store, tables=TABLES).remapped
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}}
done = prep.run(store, tables=learned)
assert (done.remapped, done.read) == (True, 1)
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
# `--remap` maps the closed roles too.
assert prep.run(store, tables=learned, remap=True).read == 2
assert row(store, "2")["seniority"] == "staff"
def test_prep_folds_the_copies(store: Store) -> None:
for n in ("1", "2"):
store.upsert_role(item(n, place="Aarhus"))
assert prep.run(store, tables=TABLES).copies == 1
assert prep.run(store, tables=TABLES, dupes=False).copies is None
# --------------------------------------------------------------- migration
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
path = tmp_path / "old.db"
store = Store(path)
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
country="DK"))
store.upsert_role(item("2"))
store.state[prep.STATE] = "old"
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_prep_due")
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
"employment", "prep_due"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE unmapped")
conn.execute("PRAGMA user_version = 21")
migrated = Store(path)
assert migrated.migrations_applied == [22]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
assert row(migrated, "2")["remote_host"] == ""
assert prep.STATE not in migrated.state
# The first prep maps every open role from those words, the vocabulary
# values the old readers stored as themselves.
prep.run(migrated, tables={})
one = row(migrated, "1")
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
("senior", "full_time", "remote", "DK")
# ------------------------------------------------------------ CLI and API
def test_app_prep_runs_and_lists_the_unmapped_words(
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
path = tmp_path / "config.yaml"
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
assert cli.main(["-c", str(path), "prep"]) == 0
assert json.loads(capsys.readouterr().out)["read"] == 1
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
assert json.loads(capsys.readouterr().out)["remapped"] is True
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
app.state.store.upsert_role(item("1", employment="Zero hours"))
prep.run(app.state.store, tables=TABLES)
with TestClient(app) as client:
found = client.get("/api/prep/unmapped").json()
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
[("employment", "acme", "zero hours", 1)]
# ------------------------------------------------------------- the tables
def test_every_host_table_maps_into_its_vocabulary() -> None:
"""Passes with no table at all; R1's and R2's files are held here."""
for host, tables in hosts.load().items():
for field, table in tables.items():
if field == "match":
assert table["mode"] in hosts.MATCH_MODES, host
values = [value for _, value in table["rules"]]
field = "employment"
else:
assert all(key == key.strip().lower() for key in table), (host, field)
values = list(table.values())
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
hosts.load()
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
importlib.invalidate_caches()
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
+37
View File
@@ -0,0 +1,37 @@
"""The ashby words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.ashby")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['REMOTE', 'EMPLOYMENT'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('onsite', 'onsite')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
@pytest.mark.parametrize(("word", "value"), [('fulltime', 'full_time'), ('parttime', 'part_time'), ('intern', 'internship'), ('contract', 'contract'), ('temporary', 'temporary')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+37
View File
@@ -0,0 +1,37 @@
"""The bamboohr words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.bamboohr")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('0', 'onsite'), ('1', 'remote'), ('2', 'hybrid')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('part time', 'part_time'), ('intern', 'internship'), ('contractor', 'contract')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+37
View File
@@ -0,0 +1,37 @@
"""The deel words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.deel")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('internship', 'internship')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+32
View File
@@ -0,0 +1,32 @@
"""The eightfold words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.eightfold")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('onsite', 'onsite'), ('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
+37
View File
@@ -0,0 +1,37 @@
"""The gem words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.gem")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('in_office', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+37
View File
@@ -0,0 +1,37 @@
"""The hibob words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.hibob")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on_site', 'onsite'), ('onsite', 'onsite')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
@pytest.mark.parametrize(("word", "value"), [('regular full time', 'full_time'), ('fixed term', 'temporary'), ('intern', 'internship')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+32
View File
@@ -0,0 +1,32 @@
"""The jibe words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.jibe")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+42
View File
@@ -0,0 +1,42 @@
"""The jobylon words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.jobylon")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['SENIORITY', 'EMPLOYMENT', 'REMOTE'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('mid level', 'mid'), ('student', 'intern'), ('entry level', 'junior'), ('executive', 'exec'), ('not applicable', '')])
def test_seniority_words(word: str, value: str) -> None:
assert hosts.SENIORITY[word] == value
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('summer job', 'temporary')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on-site', 'onsite')])
def test_remote_words(word: str, value: str) -> None:
assert hosts.REMOTE[word] == value
+32
View File
@@ -0,0 +1,32 @@
"""The lever words, mapped to the app's vocabulary."""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app import models # noqa: E402
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
hosts = importlib.import_module("app.prep.hosts.lever")
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
@pytest.mark.parametrize("table", ['EMPLOYMENT'])
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
words = getattr(hosts, table)
assert all(key == key.strip().lower() for key in words)
assert set(words.values()) <= {*VOCABULARY[table], ""}
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('fulltime', 'full_time'), ('intern', 'internship'), ('freelance', 'freelance')])
def test_employment_words(word: str, value: str) -> None:
assert hosts.EMPLOYMENT[word] == value
+12
View File
@@ -0,0 +1,12 @@
"""Oracle's word tables for prep: the host's codes, and what they mean."""
from __future__ import annotations
import importlib
oracle = importlib.import_module("app.prep.hosts.oracle")
def test_workplace_type_codes_read() -> None:
assert oracle.REMOTE == {"ora_remote": "remote", "ora_hybrid": "hybrid",
"ora_on_site": "onsite"}
+22
View File
@@ -0,0 +1,22 @@
"""Personio's word tables for prep: the feed's own words, and what they mean."""
from __future__ import annotations
import importlib
personio = importlib.import_module("app.prep.hosts.personio")
def test_seniority_words_read() -> None:
seniority = personio.SENIORITY
assert [seniority[w] for w in ("entry-level", "executive", "student")] == [
"junior", "exec", "intern"]
assert seniority["experienced"] == "" # known, names no level
assert "wizard" not in seniority
def test_employment_words_read_from_the_contract_or_the_schedule() -> None:
employment = personio.EMPLOYMENT
assert employment["working_student"] == "internship" and employment["freelance"] == "contract"
assert employment["permanent"] == ""
assert employment["full-time"] == "full_time" and employment["part-time"] == "part_time"
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff. Show more