From d04ceae7e05f5eb98305c5a57fd6cfc7ff65f923 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 11:19:22 +0200 Subject: [PATCH] Prep stage: host words stored raw, mapped centrally in app/prep after every pull upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 --- app/cli.py | 32 ++- app/jobs/__init__.py | 3 +- app/jobs/cascade.py | 32 +-- app/jobs/prep.py | 31 +++ app/jobs/pull.py | 10 +- app/models.py | 19 ++ app/prep/__init__.py | 22 +++ app/prep/country.py | 55 ++++++ app/prep/hosts/__init__.py | 61 ++++++ app/prep/hosts/platsbanken.py | 12 ++ app/prep/stage.py | 139 +++++++++++++ app/prep/words.py | 110 +++++++++++ app/static/sources.js | 41 +++- app/storage/gazetteer.py | 33 ++++ app/storage/schema.py | 84 +++++++- app/storage/store.py | 184 ++++++++++++----- app/web/api.py | 17 ++ plugins/readers/_shared.py | 6 +- tests/prepped.py | 16 ++ tests/test_architecture.py | 2 +- tests/test_companies.py | 23 +-- tests/test_dupes_storage.py | 2 +- tests/test_enrich.py | 16 +- tests/test_extract.py | 29 +-- tests/test_gazetteer.py | 37 ++-- tests/test_prep.py | 358 ++++++++++++++++++++++++++++++++++ tests/test_pull.py | 18 +- tests/test_score.py | 35 ++-- tests/test_search.py | 13 +- tests/test_storage.py | 249 +++++++++++------------ tests/test_web.py | 109 ++++++----- 31 files changed, 1468 insertions(+), 330 deletions(-) create mode 100644 app/jobs/prep.py create mode 100644 app/prep/__init__.py create mode 100644 app/prep/country.py create mode 100644 app/prep/hosts/__init__.py create mode 100644 app/prep/hosts/platsbanken.py create mode 100644 app/prep/stage.py create mode 100644 app/prep/words.py create mode 100644 tests/prepped.py create mode 100644 tests/test_prep.py diff --git a/app/cli.py b/app/cli.py index 95878a0..a5deb45 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,6 +8,7 @@ app agents list|prompt app stages [extract|summarise on|off] app dupes + app prep [unmapped] [--remap] app db init `app job` queues the work on a queue of its own and drains it here, so a @@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("dupes", help="recompute which open roles are copies of one posting") + prep = sub.add_parser( + "prep", help="map the roles' host words and fold the copies, or list the unmapped words") + prep.add_argument("action", nargs="?", choices=["unmapped"], + help="list the host words no table maps, instead of running") + prep.add_argument("--remap", action="store_true", + help="map every role again, closed ones included") + db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) return parser @@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int: return run_seniority(args) if args.command == "dupes": return run_dupes(args) + if args.command == "prep": + return run_prep(args) return init_db(args) except (FileNotFoundError, config_mod.ConfigError) as exc: print(exc, file=sys.stderr) @@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int: def run_dupes(args: argparse.Namespace) -> int: - """`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`, + """`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`, now. Prints the groups (kept roles with a copy), the copies the feed hides and the rows whose `dup_of` changed, as JSON.""" from app.storage import Store @@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int: return 0 +def run_prep(args: argparse.Namespace) -> int: + """`app prep`: the prep stage now (`app.prep.run`), printing what it did + as JSON; `--remap` maps every role again. `app prep unmapped` lists the + host words no table maps, per field and host, most roles first.""" + from app import prep + from app.storage import Store + + config = config_mod.load(args.config) + store = Store(config.database) + if args.action == "unmapped": + found = store.unmapped() + if not found: + print("every host word is mapped") + for w in found: + print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}") + return 0 + print(json.dumps(prep.run(store, remap=args.remap)._asdict())) + return 0 + + def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from app.storage import LATEST, Store diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 35ed510..ec9bd9e 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -2,11 +2,12 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. pull.py read every board and search and store the roles they yield + prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold extract.py title and description into the gate columns, on Haiku embed.py one vector per open role, for fit score.py every open role scored for every enabled search agent summarise.py about 60 words on each of the best roles, on Haiku - cascade.py pull → embed → score → extract → score → summarise, what `TASKS` + cascade.py pull → prep → embed → score → ... → summarise, what `TASKS` offers, the `pull` job a schedule fires, and that schedule stages.py whether extract and summarise run: the switch, else config.yaml sources.py what a pull reads, changed by hand: boards and searches diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index f99863b..c430545 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -1,10 +1,13 @@ """The tasks roledesk offers, and the order a pull runs them in. - pull → embed → score → enrich → embed → score → extract → score → summarise + pull → prep → embed → score → enrich → prep → embed → score → extract → score + → summarise newsdesk's cascade convention: a cascade is a tuple of task names, and the final job of one task starts the next (`step` is the callback it carries). -Nothing is alive between two tasks. A stage is addressed by its position in +Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what +the pull stored and folds the copies before anything reads the roles; after +`enrich` it maps the roles their own pages filled. A stage is addressed by its position in the tuple, not its name, since `score` runs twice. Every stage is always in the chain, and a stage that is switched off finds @@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d satisfy, is an extract stage with no batches whose final job moves straight on. The free stages go first: embed and score cost no tokens, and the score they produce is what bounds the rest. Enrich reads the best roles without -text on their own pages (`app/jobs/enrich.py`), and embed and score follow for -the roles it filled; when it filled none, `step` passes over both. Extract +text on their own pages (`app/jobs/enrich.py`), and prep, embed and score +follow for the roles it filled; when it filled none, `step` passes over all +three. Extract reads only the best roles of each enabled agent, as deep and as low as that agent's own `limits` say; the second score then applies the gates to the columns it filled. @@ -27,7 +31,7 @@ having either way. and a cascade is started by one: the `pull` job's only work is to ask for the chain. Its default schedule, `every 6h`, is declared here beside it. -`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued +`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued behind the `score` that follows a change to the ranking other than a pull's (`enrich_after`): an agent saved, a company's affinity changed. A vote moves one agent's taste a little and queues neither. @@ -40,20 +44,20 @@ from dataclasses import replace from functools import partial from typing import Any -from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs +from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs from desk.tools import agent from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job STEP = "app.jobs.cascade:step" CASCADES: dict[str, tuple[str, ...]] = { - "pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score", - "summarise"), - "enrich": ("enrich", "embed", "score"), + "pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", + "score", "summarise"), + "enrich": ("enrich", "prep", "embed", "score"), } #: What follows `enrich` for the roles it filled, passed over when it filled none. -REFILL = ("embed", "score") +REFILL = ("prep", "embed", "score") def _once(config: Any, store: Any) -> list[dict[str, Any]]: @@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = { "embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"), "score": lambda cascade: Task(kind="score", finder=_once, label="Scoring roles for every agent"), + "prep": lambda cascade: Task(kind="prep", finder=_once, + label="Mapping the roles' host words"), "summarise": _summarise, "enrich": lambda cascade: enrich.enrich_task(), } @@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str @job("enrich", "Reading the best roles on their own pages", solo=True, resource="http") def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: - """Ask for the enrich cascade, `enrich → embed → score`. Its own work is + """Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is nil, as the `pull` job's. Under `NULL` there is no queue to take it.""" if not context.task(chain("enrich")): raise RuntimeError("no queue to start the enrich cascade on") @@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any: #: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that #: is both would be listed twice by `/api/jobs/kinds`. -for _name in ("extract", "embed", "score", "summarise"): +for _name in ("extract", "embed", "score", "summarise", "prep"): TASKS[_name] = partial(task, _name) @@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"): # setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", - label="Pull every board, then embed, score, read the best roles' pages, " + label="Pull every board, then map, embed, score, read the best roles' pages, " "extract, rescore and summarise") diff --git a/app/jobs/prep.py b/app/jobs/prep.py new file mode 100644 index 0000000..013ba4b --- /dev/null +++ b/app/jobs/prep.py @@ -0,0 +1,31 @@ +"""`prep`: the stage that maps what a pull stored, `app.prep.run`. + +In the pull cascade it follows the pull, whose settle has closed what is +gone, and maps every role the pull changed, then folds the copies across +the whole database (`Store.rebuild_dupes`, which the pull's settle used to +run). After `enrich` it maps the roles their own pages filled, since only +those are due. `solo`, so two cascades never write the same rows at once. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from app import prep +from app.config import Config +from app.storage import Store +from pipeline import NULL, RunContext, job + +log = logging.getLogger(__name__) + + +@job("prep", "Mapping the roles' host words", solo=True, resource="cpu") +def run_prep(config: Config, store: Store, *, remap: bool = False, + context: RunContext = NULL) -> dict[str, Any]: + """Map the due roles, every open one when the tables changed, and fold + the copies. `remap` maps every role, closed ones included.""" + done = prep.run(store, remap=remap) + log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read, + done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "") + return done._asdict() diff --git a/app/jobs/pull.py b/app/jobs/pull.py index c0a1cab..505c58d 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. - After the closes, `Store.rebuild_dupes` recomputes which open roles are - copies of one posting, so a closed kept role hands its group on and the - roles this run added join theirs; `copies` is how many open roles it - leaves hidden as copies. + The `prep` stage that follows maps what this run stored and folds the + copies (`Store.rebuild_dupes`), so a closed kept role hands its group on + and the roles this run added join theirs. """ answered = searched = failed = incomplete = closed = 0 stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") @@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, closed += store.close_unseen("search", stale) if run_marker: release_run(store, run_marker) - copies = store.rebuild_dupes().copies roles = sum(int(result.get("roles", 0)) for result in tally.results) return {"boards": answered, "searches": searched, "failed": failed, - "incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies} + "incomplete": incomplete, "roles": roles, "closed": closed} def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int: diff --git a/app/models.py b/app/models.py index 02c90ab..cd2292c 100644 --- a/app/models.py +++ b/app/models.py @@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority) #: The levels a seniority filter may name: `SENIORITIES` without the unknown. LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level) +#: The kind of contract a posting states, as `app.prep` maps a host's word. +#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the +#: readers' own tests, since a plugin imports nothing from `app`. +Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship", + "freelance"] +EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e) + #: Whether a posting says it sponsors a work permit. '' is "it does not say". Permit = Literal["", "sponsors", "no_sponsorship"] PERMITS: tuple[str, ...] = get_args(Permit) @@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool: _POSTED_FLOOR = "2015-01-01T00:00:00+00:00" +def host_word(value: Any) -> str: + """A host's word as `roles.*_host` stores it: stripped text, a bool as + 'true' or 'false', '' for nothing. What `app.prep` maps.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + return str(value).strip() + + def posted_host(raw: dict[str, Any]) -> str: """The host a role's dates came from, read the way `ats` and `search` both put it in `Item.raw`: `board` for an ATS board (`greenhouse`, @@ -797,6 +814,8 @@ class Role(Model): seniority_model: str = "" seniority_stated: str = "" seniority_title: str = "" + #: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`. + employment: str = "" function: str = "" permit: Permit = "" #: ISO 639-1 codes the posting says are required, lowercase. diff --git a/app/prep/__init__.py b/app/prep/__init__.py new file mode 100644 index 0000000..7d64ac8 --- /dev/null +++ b/app/prep/__init__.py @@ -0,0 +1,22 @@ +"""Prep: every decision about a role that needs a rule or data across roles. + +Readers scrape and hand on the host's own words (`Store.upsert_role` keeps +them in `roles.*_host`); prep maps them, once, centrally, between the pull +and embedding. One door: `from app.prep import ...`. + + words.py seniority, employment and remote: the host's table, the + vocabulary itself, the shared remote words + country.py the country from the resolved places, else the host's + hosts/ one table file per reader (`hosts.TABLES`) + stage.py `run`: the stage over the due roles, `unmapped` and the fold +""" + +from __future__ import annotations + +from app.prep.country import country +from app.prep.hosts import TABLES +from app.prep.stage import STATE, Prepped, company, fingerprint, run +from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority + +__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country", + "employment", "fingerprint", "lookup", "remote", "run", "seniority"] diff --git a/app/prep/country.py b/app/prep/country.py new file mode 100644 index 0000000..3caf32b --- /dev/null +++ b/app/prep/country.py @@ -0,0 +1,55 @@ +"""A role's country, from its places, else from the one its host states.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from app import places + + +def of_host(gaz: places.Gazetteer, stated: str) -> str | None: + """The ISO code a host's country names, in the host's spelling: a code + ("DK", "dk"), or a name the gazetteer has as an alias of a country + ("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one + country. '' for nothing stated, None for a text naming no country.""" + text = (stated or "").strip() + if not text: + return "" + if ("country", text.upper()) in gaz.by_key or text.upper() == "EU": + return text.upper() + named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key + for p in gaz.aliases.get(places.fold(text), ()) + if gaz.places[p].kind == "country" + or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")} + if len(named) == 1: + return named.pop() + found = {gaz.country(p) for p in gaz.resolve(text)} - {""} + return found.pop() if len(found) == 1 else None + + +def _in_europe(gaz: places.Gazetteer, code: str) -> bool: + if code == "EU": + return True + europe = gaz.by_key.get(("continent", "europe")) + place_id = gaz.by_key.get(("country", code)) + return europe is not None and place_id is not None and europe in gaz.ancestors(place_id) + + +def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str: + """The one country the role's resolved places lie in, continents aside; + Europe alone is `EU`; else, when they + lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin, + London" and the country gate reads as Europe); else the host's stated + country (`of_host`); else ''.""" + ids = list(place_ids) + # A continent beside a country is the wider unit a line adds ("Home + # based - EMEA; London" is GB), as `country_of` had it; alone, Europe + # is `EU`. + found = {gaz.country(p) for p in ids} - {""} + if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places): + found = {"EU"} + if len(found) == 1: + return found.pop() + if len(found) > 1 and all(_in_europe(gaz, code) for code in found): + return "EU" + return of_host(gaz, host_country) or "" diff --git a/app/prep/hosts/__init__.py b/app/prep/hosts/__init__.py new file mode 100644 index 0000000..518890a --- /dev/null +++ b/app/prep/hosts/__init__.py @@ -0,0 +1,61 @@ +"""Each host's own words, one module per reader, named as the reader is. + +A module defines any of three dicts, keyed by the host's word lowercased: + +- `SENIORITY`, values in `app.models.SENIORITIES` +- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS` +- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite) + +A value of "" is a word the host uses that names no value ("Experienced", +"Permanent"): known, so it is not listed as unmapped. + +A host whose employment word is a code of several parts (`permanent_full_time`, +`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order +after `EMPLOYMENT` with the first hit winning, and must then say how a +fragment matches in `MATCH_MODE`: "substring" (the fragment is in the +lowercased word) or "token" (it equals one of the word's parts split at `_` +and `-`, so `ft` does not match inside `software`). `TABLES` loads every +module of this package by name, so a table is added by adding its file. +""" + +from __future__ import annotations + +import importlib +import pkgutil +from typing import Any + +#: The dict a module names, by the field it maps. +FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"} + +#: How a `MATCH` fragment may match. +MATCH_MODES = ("substring", "token") + + +def load() -> dict[str, dict[str, Any]]: + """`host -> field -> {word: value}` for every module of the package, the + words lowercased and stripped. A host without a table for a field has + no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules": + [[fragment, value], ...]}`; a module with `MATCH` and no valid + `MATCH_MODE` fails the import by name.""" + tables: dict[str, dict[str, Any]] = {} + for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name): + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{__name__}.{info.name}") + for field, name in FIELDS.items(): + table = getattr(module, name, None) + if isinstance(table, dict): + tables.setdefault(info.name, {})[field] = { + str(k).strip().lower(): str(v) for k, v in table.items()} + rules = getattr(module, "MATCH", None) + if rules is not None: + mode = getattr(module, "MATCH_MODE", None) + if mode not in MATCH_MODES: + raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE " + f"is {mode!r}, not one of {', '.join(MATCH_MODES)}") + tables.setdefault(info.name, {})["match"] = { + "mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]} + return tables + + +TABLES = load() diff --git a/app/prep/hosts/platsbanken.py b/app/prep/hosts/platsbanken.py new file mode 100644 index 0000000..ac510fc --- /dev/null +++ b/app/prep/hosts/platsbanken.py @@ -0,0 +1,12 @@ +"""Platsbanken's words, for prep.""" + +#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats" +#: is in the fixtures; the others are JobTech's other labels as the site +#: writes them, and a label not here is listed as unmapped. +REMOTE: dict[str, str] = { + "arbete på plats": "onsite", "på plats": "onsite", + "hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid", + "delvis distans": "hybrid", + "distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote", + "distans": "remote", +} diff --git a/app/prep/stage.py b/app/prep/stage.py new file mode 100644 index 0000000..02aa499 --- /dev/null +++ b/app/prep/stage.py @@ -0,0 +1,139 @@ +"""The prep stage: map what the readers handed on, then fold the copies. + +A pull stores the host's own words and place texts (`Store.upsert_role`) +and marks the role due. `run` maps every due role: its company through +`company_aliases`, its places against the gazetteer, its level, contract +and remote through the host's tables (`app.prep.words`), and its country +from the places (`app.prep.country`). When the tables changed since the +last run (`fingerprint`, kept in state as `prep:tables`), every open role +is mapped again from its stored words, with no new pull. Then the words no +table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the +copies across the whole database. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from typing import Any, NamedTuple + +from app import places +from app.models import company_key +from app.models import company_fold +from app.prep import words +from app.prep.country import country, of_host +from app.prep.hosts import TABLES +from app.prep.words import Tables +from app.storage import Store + +#: Raised when the rules in this package change the way a stored word maps, +#: so every open role is mapped again as it is after a table changes. +VERSION = 1 + +#: The state key the last run's `fingerprint` is kept under. +STATE = "prep:tables" + + + +class Prepped(NamedTuple): + """What one run did: roles read, rows written, whether every role was + mapped again, words left unmapped, and the open copies after the fold + (None when it was not run).""" + + read: int + written: int + remapped: bool + unmapped: int + copies: int | None + + +def fingerprint(tables: Tables = TABLES) -> str: + """A hash of every host table, the shared remote words and `VERSION`.""" + blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS}, + sort_keys=True) + return hashlib.sha1(blob.encode()).hexdigest() + + +def company(folds: Mapping[str, Sequence[str]], name: str) -> str: + """The key a role from `name` belongs to, as `Store.resolve_company` + has it: the company an alias names, the name's own key first when + several share the fold, else `company_key(name)`.""" + own = company_key(name) + keys = folds.get(company_fold(name), ()) + return own if own in keys or not keys else keys[0] + + +def _places(raw: Any) -> list[Any]: + try: + found = json.loads(raw) if isinstance(raw, str) else raw + except ValueError: + return [] + return found if isinstance(found, list) else [] + + +def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]], + place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]: + """One role as `Store.write_prep` takes it, from a `Store.prep_rows` row.""" + host = str(row["host"] or "") + stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables) + return { + "uid": row["uid"], + "due": row["prep_due"], + "company_key": company(folds, row["company"]) if row["company"] else row["company_key"], + "seniority_stated": stated, + "seniority_title": titled, + "employment": words.employment(host, row["employment_host"], tables), + "remote": words.remote(host, row["remote_host"] or None, (), "", tables), + "remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"], + tables), + "country": country(gaz, place_ids, row["country_host"]), + } + + +def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool: + level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"] + return bool(row["prep_due"]) or level != row["seniority"] \ + or (found["remote"] and found["remote"] != row["remote"]) \ + or (found["remote_places"] and not row["remote"]) \ + or (found["country"] and found["country"] != row["country"]) \ + or any(found[c] != row[c] for c in ("company_key", "seniority_stated", + "seniority_title", "employment")) + + +def unknown(field: str, host: str, word: str, gaz: places.Gazetteer, + tables: Tables = TABLES) -> bool: + """Whether no table, vocabulary or gazetteer knows this host word.""" + if field == "country": + return of_host(gaz, word) is None + if field == "remote" and word in ("true", "false"): + return False + return words.lookup(field, host, word, tables) is None + + +def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, + dupes: bool = True) -> Prepped: + """Map the due roles, or every open one when the tables changed, or + every role with `remap`; list the unmapped words; fold the copies.""" + mark = fingerprint(tables) + whole = remap or store.state.get(STATE) != mark + rows = store.prep_rows("due") + if whole: + due = {r["uid"] for r in rows} + rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due] + ids = store.resolve_role_places([r["uid"] for r in rows]) + gaz = store.gazetteer() + folds = store.company_folds() + written = [] + for row in rows: + found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables) + if _changed(row, found): + written.append(found) + store.write_prep(written) + left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)] + store.replace_unmapped(left) + if whole: + store.state[STATE] = mark + copies = store.rebuild_dupes().copies if dupes else None + return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left), + copies=copies) diff --git a/app/prep/words.py b/app/prep/words.py new file mode 100644 index 0000000..4d71dc8 --- /dev/null +++ b/app/prep/words.py @@ -0,0 +1,110 @@ +"""A host's word to the app's vocabulary: seniority, employment and remote. + +Each field is looked up the same way: the host's table in `hosts`, keyed by +the word lowercased, then a word that already is a value of the vocabulary +(what readers stored before they handed on the host's words, and what a +host with the app's own words sends), then, for remote, the shared word map. +An employment word may also match a host's `MATCH` fragments. A word none +of these knows is unmapped: it maps to '' and `lookup` says so, +so the prep stage can list it. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of +from app.prep.hosts import TABLES +from app.seniority import from_title + +#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`). +Tables = Mapping[str, Mapping[str, Any]] + +VOCABULARIES: dict[str, tuple[str, ...]] = { + "seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE} + +#: How hosts without a table of their own write remote, after `_squash`: +#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office"). +REMOTE_WORDS: dict[str, str] = { + "remote": "remote", "fullyremote": "remote", "remotefirst": "remote", + "hybrid": "hybrid", + "onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite", + "officebased": "onsite", "onpremise": "onsite", +} + + +def _match(rules: Mapping[str, Any], key: str) -> str | None: + """The value of the first `MATCH` fragment `key` holds, by its mode.""" + parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set() + for fragment, value in rules["rules"]: + if fragment in parts if rules["mode"] == "token" else fragment in key: + return str(value) + return None + + +def _squash(word: str) -> str: + return "".join(c for c in word.lower() if c not in " -_") + + +def lookup(field: str, host: str, word: str, + tables: Tables = TABLES) -> str | None: + """The value `word` maps to for `host`, '' for a known word that names + none, None for a word nothing knows. An empty word is ''.""" + key = (word or "").strip().lower() + if not key: + return "" + table = tables.get(host, {}) + found = table.get(field, {}).get(key) + if found is None and field == "employment" and "match" in table: + found = _match(table["match"], key) + if found is not None: + return found + if key in VOCABULARIES[field]: + return key + if field == "remote": + return REMOTE_WORDS.get(_squash(key)) + return None + + +def seniority(host: str, word: str, title: str, + tables: Tables = TABLES) -> tuple[str, str]: + """`(stated, from the title)`: the host's word through its table, and + `seniority.from_title`. `roles.seniority` is the model's level, else the + stated one, else the title's (`Store.write_prep`).""" + return lookup("seniority", host, word, tables) or "", from_title(title) + + +def employment(host: str, word: str, + tables: Tables = TABLES) -> str: + return lookup("employment", host, word, tables) or "" + + +def _place_text(place: Any) -> str: + return str((place.get("text") if isinstance(place, Mapping) else place) or "") + + +def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", + tables: Tables = TABLES) -> str: + """How the role is worked, one of `REMOTE` or ''. + + `value` is the host's own (a bool, or its word; `host_word`'s 'true' and + 'false' are bools). True is remote. False says nothing more than the + places do: SmartRecruiters, Ashby, Recruitee and Workable send it for + every role that is not remote, hybrid ones included. A word goes + through the host's table, else the shared map. When the value says + nothing, the place line or the place texts are read for a word + (`remote_of`: "Hybrid, Berlin"), and last a place the reader marked + remote is remote.""" + if value is True or value == "true": + return "remote" + said = value not in (None, False, "false") + if said and (found := lookup("remote", host, str(value), tables)): + return found + places = list(places) + if found := remote_of(line or "; ".join(map(_place_text, places))): + return found + if any(isinstance(p, Mapping) and p.get("remote") is True for p in places): + return "remote" + return "" diff --git a/app/static/sources.js b/app/static/sources.js index c986b8b..e306ae5 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -19,7 +19,9 @@ * Under the Follow form of Company boards, a fold lists what each reader * declares (`/api/readers`): how its list arrives, where the text is, what * it filters at the source, how it states places, and the robots.txt - * decision taken when it was built. + * decision taken when it was built. A second fold lists the host words no + * table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and + * host, most roles first; it is left out when there are none. */ (function () { "use strict"; @@ -320,6 +322,38 @@ $("#board-form").after(fold); } + /** The host words prep could not map, one row per field and host. */ + function renderUnmapped(words) { + if (!words.length) return; + const groups = new Map(); + for (const w of words) { + const key = `${w.field} · ${w.host || "no host"}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(w); + } + const fold = document.createElement("details"); + fold.className = "prune"; + fold.id = "unmapped"; + const summary = document.createElement("summary"); + summary.textContent = `Host words no table maps (${words.length})`; + const list = document.createElement("div"); + list.className = "srows"; + for (const [key, found] of groups) { + const row = document.createElement("div"); + row.className = "srow"; + const name = document.createElement("span"); + name.className = "name"; + name.textContent = key; + const sub = document.createElement("span"); + sub.className = "sub"; + sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · "); + row.append(name, sub); + list.appendChild(row); + } + fold.append(summary, list); + ($("#readers") || $("#board-form")).after(fold); + } + // ------------------------------------------------------ one section shown // Sources shows one section: the one the fragment names or contains, or the @@ -360,7 +394,10 @@ document.addEventListener("DOMContentLoaded", async () => { if (!$("#searches-list")) return; - if ($("#board-form")) api("/api/readers").then(renderReaders, () => {}); + if ($("#board-form")) { + api("/api/readers").then(renderReaders, () => {}) + .then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {}); + } try { await load(); } catch (err) { diff --git a/app/storage/gazetteer.py b/app/storage/gazetteer.py index ae3386f..dd987a2 100644 --- a/app/storage/gazetteer.py +++ b/app/storage/gazetteer.py @@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here. from __future__ import annotations +import json import sqlite3 from collections.abc import Iterable, Mapping, Sequence from typing import Any @@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str, conn.executemany(_INSERT, rows(gaz, uid, place_texts)) +def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None: + """Replace one role's own places with these texts, unresolved: what a + pull stores, for `app.prep` to resolve (`relink_uids`).""" + conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,)) + conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None) + for position, text in enumerate(place_texts)]) + + +def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer, + uids: Sequence[str]) -> dict[str, list[int]]: + """Resolve the own places of these roles, writing only those that now + resolve differently. Returns `uid -> place ids` in the host's order for + every role of `uids` with places, what prep reads the country from.""" + held: dict[str, dict[tuple[int, str], set[int | None]]] = {} + for start in range(0, len(uids), 5000): + chunk = json.dumps(list(uids[start:start + 5000])) + for uid, position, text, place_id in conn.execute( + "SELECT uid, position, text, place_id FROM role_places WHERE url = '' " + "AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid", + (chunk,)): + held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id) + found: dict[str, list[int]] = {} + changed = [] + for uid, rows_of in held.items(): + now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of} + if now != rows_of: + changed.append((uid, [text for _, text in sorted(rows_of)])) + found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])] + write_many(conn, gaz, changed) + return found + + def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer, by_uid: Iterable[tuple[str, Sequence[str]]]) -> int: """`write` for many roles in one statement each way. Returns rows written.""" diff --git a/app/storage/schema.py b/app/storage/schema.py index 60b0cb6..4b08f93 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -18,7 +18,9 @@ import sqlite3 from collections.abc import Callable from app import places -from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host +from app.models import ( + SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host, +) from app.seniority import from_title from app.storage import gazetteer @@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles ( teaser TEXT NOT NULL DEFAULT '', seniority_model TEXT NOT NULL DEFAULT '', seniority_stated TEXT NOT NULL DEFAULT '', - seniority_title TEXT NOT NULL DEFAULT '' + seniority_title TEXT NOT NULL DEFAULT '', + seniority_host TEXT NOT NULL DEFAULT '', + employment_host TEXT NOT NULL DEFAULT '', + remote_host TEXT NOT NULL DEFAULT '', + country_host TEXT NOT NULL DEFAULT '', + employment TEXT NOT NULL DEFAULT '', + prep_due INTEGER NOT NULL DEFAULT 0 ); +-- The *_host columns hold the host's own words as a reader handed them on +-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps +-- them into seniority_stated, employment, remote and country. prep_due is above 0 +-- on a role whose words, company, title or places changed since prep last +-- mapped it (Store.upsert_role), which is what the next prep reads. -- roles.teaser is the short text a list carries, kept until the full text -- arrives and embedded while there is none; a later empty teaser does not -- wipe it. seniority_model, seniority_stated and seniority_title are the @@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors ( embedded_at TEXT NOT NULL ); +-- A host word no table of app/prep/hosts maps, per field (seniority, +-- employment, remote, country) and host: how many open roles carry it and +-- when prep first met it. Rewritten by every prep; a word added to a table +-- leaves at the next one. +CREATE TABLE IF NOT EXISTS unmapped ( + field TEXT NOT NULL, + host TEXT NOT NULL, + word TEXT NOT NULL, + roles INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + PRIMARY KEY (field, host, word) +); + -- The string map a desk plugin is handed as self.state. One namespace shared -- by every plugin, so a plugin prefixes its keys. CREATE TABLE IF NOT EXISTS state ( @@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid); CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid); CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id); +-- The roles the next prep maps. Partial: after a prep there are none. +CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0; """ #: The table whose absence marks a file that has never been a roledesk @@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT ''; (21, "companies.all_roles: a company tag that reads its boards with the agents' wide " "reach, given to every favourite and taking over a board's {\"reach\": \"all\"}", lambda conn: _add_all_roles(conn)), + # A Python step: the host words are read out of raw's JSON. + (22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own " + "words are stored and app.prep maps them", lambda conn: _add_host_words(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None: "to tag; it reads the agents' reach now", target, key) +_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment") + +_V22 = """ +CREATE TABLE IF NOT EXISTS unmapped ( + field TEXT NOT NULL, + host TEXT NOT NULL, + word TEXT NOT NULL, + roles INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + PRIMARY KEY (field, host, word) +); +DELETE FROM state WHERE key = 'prep:tables'; +""" + + +def _add_host_words(conn: sqlite3.Connection) -> None: + """Migration 22's step. The columns and `unmapped`, and the host words + of every role filled from its `raw` where it has them. Before this, + readers mapped `seniority` and `employment` in their own tables, so what + `raw` holds there is mostly a value of the vocabulary already, which + `app.prep` takes as it is. `remote` and `country` are the host's as + given. The `prep:tables` fingerprint goes, so the first prep maps every + open role from these words. Runs twice without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} + for column in _V22_COLUMNS: + if column not in columns: + conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''") + if "prep_due" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0") + for statement in _statements(_V22): + conn.execute(statement) + updates = [] + for uid, raw in conn.execute( + "SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL " + "OR json_extract(raw, '$.employment') IS NOT NULL " + "OR json_extract(raw, '$.remote') IS NOT NULL " + "OR json_extract(raw, '$.country') IS NOT NULL").fetchall(): + try: + given = json.loads(raw or "{}") + except ValueError: + continue + if isinstance(given, dict): + updates.append((*(host_word(given.get(k)) for k in ( + "seniority", "employment", "remote", "country")), uid)) + conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, " + "remote_host = ?, country_host = ? WHERE uid = ?", updates) + log.info("migration 22: host words from raw for %d roles", len(updates)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index dd0360b..b5b9932 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,13 +22,12 @@ import numpy as np from desk.models import Item from app import dupes, places from app.models import ( - AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application, + AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application, ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace, RoleRow, ScoredRole, Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key, - posted_at, posted_host, remote_of, title_key, + host_word, posted_at, posted_host, remote_of, title_key, ) -from app.seniority import from_title from app.storage import gazetteer, schema #: Text columns a plugin may fill from `Item.raw` under the same name. An @@ -36,7 +35,17 @@ from app.storage import gazetteer, schema #: says less than `extract` found keeps what `extract` found. `function` is #: not here: it is not a board field (see the note by `keep_filled` below). #: Nor is `seniority`, which is the title's level unless a board states one. -_RAW_TEXT = ("place", "country", "currency") +_RAW_TEXT = ("place", "currency") + +#: The host's own words a reader hands on in `Item.raw`, stored as +#: `_host` (`host_word`) for `app.prep` to map. `seniority_host` is +#: rewritten every pull, as the host's level always was; the others are kept +#: filled, since a role's own page (`enrich`) may state what its listing +#: does not. +_HOST_WORDS = ("seniority", "employment", "remote", "country") +_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host") +#: What prep maps from that a pull rewrites whatever it brings. +_PREP_OVERWRITTEN = ("title", "seniority_host") _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own @@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str: return moment.astimezone(UTC).isoformat(timespec="seconds") -def _remote(value: Any) -> str: - """A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything - unrecognised is '' (unknown), which `extract` may fill later.""" - if value is True: - return "remote" - text = str(value or "").strip().lower().replace("-", "").replace(" ", "") - text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text) - return text if text in REMOTE else "" +def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]: + """One role's own place texts as stored, in the host's order.""" + return list(dict.fromkeys(r[0] for r in conn.execute( + "SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid", + (uid,)))) def _like(text: str) -> str: @@ -607,10 +613,19 @@ class Store: """Store one posting. Returns True when it was new. `first_seen` and `created_at` are written on insert only; `last_seen` on - every call, and a role seen again is open again. The gate columns come + every call, and a role seen again is open again. The raw columns come from `item.raw` under their own names, with `company` falling back to - the subscription's and `country` to what `place` names; an empty value - never overwrites a filled column. + the subscription's; an empty value never overwrites a filled column. + + Nothing is mapped here: that is `app.prep`, which runs after every + pull. The host's words go to `seniority_host`, `employment_host`, + `remote_host` and `country_host` (`host_word`), `company_key` is + written unresolved on a new role only, and the mapped columns + (`seniority_stated`, `seniority_title`, `seniority`, `employment`, + `remote`, `country`) are left as prep last wrote them. `prep_due` + is 1 on a new role and grows by one when the title, the company, a + host word or the place texts changed or the role reopens, which + is what the next prep reads. `posted_at` (see `app.models.posted_at`) is set from this pull's own `published_at` and `stamp`; on an update the `ON CONFLICT` clause below @@ -627,20 +642,14 @@ class Store: alone for good, whatever the plugin's raw guess now says — see `set_extracted` for the other half of the rule. - `seniority` is the first filled of three levels. `seniority_model` - is extract's (`set_extracted`) and never written here. - `seniority_stated` is the host's word as `raw["seniority"]` carries - it, already in `SENIORITIES` (anything else is ''), and - `seniority_title` is `seniority.from_title`; both are rewritten every - pull. The classifier is called here rather than in a plugin, so a - role from any source gets it, the way `country` comes from `place`. - The role's places (`role_places`) are the texts of `raw["places"]`, - else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved - against the gazetteer; `place` is the line shown, the texts joined - when `raw` has none. Like `place`, a pull with no place leaves them - as they are. `place_fold` is written with `place`, under the same - keep-filled rule. `teaser` is kept-filled too. + else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved + when they differ from the stored ones; prep resolves them. Texts that + did not change keep their resolved rows, so the feed's Place filter + stays right between a pull and its prep. `place` is the line shown, + the texts joined when `raw` has none. Like `place`, a pull with no + place leaves them as they are. `place_fold` is written with `place`, + under the same keep-filled rule. `teaser` is kept-filled too. A role is embedded from its description, else its teaser, else its title (`app.jobs.embed.document`). When a pull brings a better one of @@ -660,12 +669,11 @@ class Store: "source": item.source, "source_id": item.source_id, "subscription_id": subscription.id if subscription else None, - "company_key": self.resolve_company(company), + "company_key": company_key(company), "company": company, "title": item.title, "title_key": title_key(item.title), "place_key": place_key(place), - "remote": _remote(raw.get("remote")), "description": item.body, "feed_text": item.excerpt, "original_url": item.location, @@ -677,30 +685,26 @@ class Store: "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} + row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS} row["place"] = place - row["country"] = row["country"] or country_of(place) - row["remote"] = row["remote"] or remote_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} row["function"] = str(raw.get("function") or "") - stated = str(raw.get("seniority") or "") - row["seniority_stated"] = stated if stated in SENIORITIES else "" - row["seniority_title"] = from_title(item.title) - row["seniority"] = row["seniority_stated"] or row["seniority_title"] row["teaser"] = str(raw.get("teaser") or "") row["place_fold"] = places.fold(row["place"]) # A board's own pay line, one per pull like title or published_at — # not kept-filled like _RAW_TEXT: a board that stops stating pay # must be able to clear it, not leave a stale line behind. row["salary_text"] = str(raw.get("salary_text") or "") + row["prep_due"] = 1 - keep_filled = ("company", "company_key", "place_key", "remote", "description", - "feed_text", "original_url", "teaser", *_RAW_TEXT) + keep_filled = ("company", "place_key", "description", "feed_text", "original_url", + "teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS) updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled] updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})" for c in (*_RAW_INT, "subscription_id")] updates += [f"{c}=excluded.{c}" for c in ("title", "title_key", "published_at", "raw", "last_seen", - "salary_text", "seniority_stated", "seniority_title")] + "salary_text", "seniority_host")] updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)") # Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh # pull's classifier guess overwrite what extract decided, which is @@ -711,11 +715,17 @@ class Store: updates.append("function=CASE WHEN roles.extracted_at IS NULL " "THEN COALESCE(NULLIF(excluded.function, ''), roles.function) " "ELSE roles.function END") - # The model's level, else the host's, else the title's. - updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), " - "NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)") updates.append("place_fold=CASE WHEN excluded.place != '' " "THEN excluded.place_fold ELSE roles.place_fold END") + # One more than before when what prep maps from changed, or the role + # reopens, so a prep that read the old values leaves it due + # (`Store.prepped`). `roles.` is the stored row here, `excluded.` + # what this pull brought. + changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN] + changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})" + for c in ("company", *_KEPT_HOST_WORDS)] + updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} " + "THEN roles.prep_due + 1 ELSE roles.prep_due END") updates.append("closed_at=NULL") columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) @@ -725,8 +735,11 @@ class Store: conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) - if place_texts: - gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts) + if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts): + gazetteer.write_texts(conn, row["uid"], place_texts) + if before is not None: + conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?", + (row["uid"],)) # `embed` encodes a role once, from the best text it has. When a # later pull brings a better one, as a host that states the text # on the role's own page does, the vector goes, so the next @@ -1253,6 +1266,89 @@ class Store: remotes) return len(countries) + len(remotes) + # ----------------------------------------------------------------- prep + + def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]: + """What `app.prep` maps a role from, and what it holds now: the roles + due (`prep_due` above 0), every open role, or every role.""" + where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope] + return self._rows( + f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place, + seniority_host, employment_host, remote_host, country_host, + seniority_model, seniority_stated, seniority_title, seniority, + employment, remote, country, + json_extract(raw, '$.places') AS raw_places + FROM roles WHERE {where}""") + + def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]: + """Resolve these roles' own places against the gazetteer, writing the + rows that resolve differently; `uid -> place ids` for each that has + places (`gazetteer.relink_uids`).""" + with self.transaction() as conn: + return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids)) + + def company_folds(self) -> dict[str, list[str]]: + """`fold -> company keys` of `company_aliases`, sorted: what + `resolve_company` asks one name at a time, for prep's whole batch.""" + out: dict[str, list[str]] = {} + for r in self._rows("SELECT fold, company_key FROM company_aliases " + "ORDER BY fold, company_key"): + out.setdefault(str(r["fold"]), []).append(str(r["company_key"])) + return out + + def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int: + """Write what prep mapped, one dict per role with `uid`, `due` (the + `prep_due` it read), `company_key`, `seniority_stated`, + `seniority_title`, `employment`, `remote`, `remote_places` and + `country`. `seniority` is the model's level, else the stated, else + the title's. An empty `remote` or `country` leaves the column as it + is, since `extract` fills an empty one. `remote_places`, what the + place texts say, only fills an empty column: it is a guess from a + line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull + changed the role since it was read. Returns the rows written.""" + with self.transaction() as conn: + return conn.executemany( + """UPDATE roles SET company_key = :company_key, + seniority_stated = :seniority_stated, + seniority_title = :seniority_title, + seniority = COALESCE(NULLIF(seniority_model, ''), + NULLIF(:seniority_stated, ''), :seniority_title), + employment = :employment, + remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''), + :remote_places), + country = COALESCE(NULLIF(:country, ''), country), + prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END + WHERE uid = :uid""", mapped).rowcount + + def host_words(self) -> list[tuple[str, str, str, int]]: + """`(field, host, word, open roles)` for every host word an open role + carries, the word lowercased: what prep checks against its tables.""" + found: list[tuple[str, str, str, int]] = [] + for field in ("seniority", "employment", "remote", "country"): + found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows( + f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles + WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")] + return found + + def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None: + """Make `unmapped` these `(field, host, word, roles)`, keeping the + `first_seen` of a word already listed.""" + stamp = self.now() + with self.transaction() as conn: + seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute( + "SELECT field, host, word, first_seen FROM unmapped")} + conn.execute("DELETE FROM unmapped") + conn.executemany( + "INSERT INTO unmapped (field, host, word, roles, first_seen) " + "VALUES (?, ?, ?, ?, ?)", + [(*r, seen.get(r[:3], stamp)) for r in rows]) + + def unmapped(self) -> list[dict[str, Any]]: + """The host words no table maps, by field and host, most roles first.""" + return [dict(r) for r in self._rows( + "SELECT field, host, word, roles, first_seen FROM unmapped " + "ORDER BY field, host, roles DESC, word")] + # -------------------------------------------------------------- extract def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]: diff --git a/app/web/api.py b/app/web/api.py index 27e1589..8cf95c1 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]: for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))] +class UnmappedOut(BaseModel): + """A host word no table of `app/prep/hosts` maps (`Store.unmapped`).""" + + field: str + host: str + word: str + roles: int + first_seen: str + + +@router.get("/prep/unmapped") +def list_unmapped(store: StoreDep) -> list[UnmappedOut]: + """The host words the last prep could not map, per field and host, most + roles first: what a table in `app/prep/hosts` should learn.""" + return [UnmappedOut(**w) for w in store.unmapped()] + + class SearchBody(BaseModel): provider: str name: str = "" diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index 6894556..c7cf67a 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -197,9 +197,9 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)" -#: The employment words a row's `employment` may hold, for every host. A -#: host's table maps its own words onto these; a word that names none of them -#: (a permanent contract says nothing about hours) maps to "". +#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a +#: host's employment word onto, kept for the readers' tests since a plugin +#: imports nothing from `app`. `tests/test_prep.py` holds the two equal. EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance") _REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE) diff --git a/tests/prepped.py b/tests/prepped.py new file mode 100644 index 0000000..03557b6 --- /dev/null +++ b/tests/prepped.py @@ -0,0 +1,16 @@ +"""`upsert`: a role stored as a pull stores it and then mapped, as the prep +stage after the pull maps it. For tests of what the feed, the gates and the +pages read, which are prep's values, not the raw ones `upsert_role` keeps.""" + +from __future__ import annotations + +from typing import Any + +from app import prep +from app.storage import Store + + +def upsert(store: Store, item: Any, subscription: Any = None) -> bool: + new = store.upsert_role(item, subscription) + prep.run(store, dupes=False) + return new diff --git a/tests/test_architecture.py b/tests/test_architecture.py index c3dc326..58d46d7 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1] #: no database, so storage may use them. `main` assembles the app from #: everything below it; `cli` and `__main__` are the entry points and may #: reach anything. -LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli", +LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli", "__main__"] APP = "app" EXTERNAL = {"desk", "pipeline", "auth"} diff --git a/tests/test_companies.py b/tests/test_companies.py index e17a0bb..7ba9a86 100644 --- a/tests/test_companies.py +++ b/tests/test_companies.py @@ -13,6 +13,7 @@ from app.models import company_fold from app.storage import Store from app.storage.schema import _add_company_aliases from desk.models import Item +from prepped import upsert def undo_v20(conn: sqlite3.Connection) -> None: @@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None: def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None: store.add_company("Ørsted") - store.upsert_role(posting("1", "Orsted")) - store.upsert_role(posting("2", "ORSTED A/S")) - store.upsert_role(posting("3", "Ørsted")) - store.upsert_role(posting("4", "Nobody")) + upsert(store, posting("1", "Orsted")) + upsert(store, posting("2", "ORSTED A/S")) + upsert(store, posting("3", "Ørsted")) + upsert(store, posting("4", "Nobody")) assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"] assert [c.key for c in store.companies()] == ["ørsted"] @@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None: def test_an_added_alias_finds_the_company(store: Store) -> None: store.add_company("Ørsted") store.add_alias("ørsted", "Dong Energy") - store.upsert_role(posting("1", "Dong Energy A/S")) + upsert(store, posting("1", "Dong Energy A/S")) assert company_of(store, "1") == "ørsted" with pytest.raises(KeyError): store.add_alias("nope", "X") @@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None: assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(store.companies()) == 3 # The company with the name's own key wins when a role names it. - store.upsert_role(posting("1", "Orsted")) + upsert(store, posting("1", "Orsted")) assert company_of(store, "1") == "orsted" def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None: store.add_company("Ørsted") store.add_company("Orsted") - store.upsert_role(posting("1", "Orsted")) - store.upsert_role(posting("2", "Ørsted")) + upsert(store, posting("1", "Orsted")) + upsert(store, posting("2", "Ørsted")) sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted", board="greenhouse") a, b = store.save_agent("A"), store.save_agent("B") @@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None: assert store.duplicate_companies() == [] assert store.company_aliases() == {"ørsted": ["orsted"]} # A later role from the gone spelling lands on the kept company. - store.upsert_role(posting("3", "Orsted")) + upsert(store, posting("3", "Orsted")) assert company_of(store, "3") == "ørsted" with pytest.raises(ValueError): store.merge_companies("ørsted", "ørsted") @@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> "VALUES (?, ?, '', '')", (key, name)) conn.execute("PRAGMA user_version = 19") reopened = Store(path) - assert reopened.migrations_applied == [20, 21] + assert reopened.migrations_applied == [20, 21, 22] assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(reopened.companies()) == 3 @@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp conn.execute("ALTER TABLE companies DROP COLUMN all_roles") conn.execute("PRAGMA user_version = 20") reopened = Store(path) - assert reopened.migrations_applied == [21] + assert reopened.migrations_applied == [21, 22] assert {c.key: c.all_roles for c in reopened.companies()} == { "mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0} params = {s.target: s.params for s in reopened.subscriptions(source="ats")} diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index 86ece58..66c823a 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [18, 19, 20, 21] + assert migrated.migrations_applied == [18, 19, 20, 21, 22] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_enrich.py b/tests/test_enrich.py index 541b95d..ed2190c 100644 --- a/tests/test_enrich.py +++ b/tests/test_enrich.py @@ -250,16 +250,16 @@ class _Context: return t -def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None: +def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None: context = _Context() empty = Tally(results=({"filled": 0},)) - out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3}, + out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4}, context=context) - assert out["next"] == "extract" and context.made[0].carries["at"] == 6 + assert out["next"] == "extract" and context.made[0].carries["at"] == 8 out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})), - {"cascade": "pull", "task": "enrich", "at": 3}, context=context) - assert out["next"] == "embed" and context.made[1].carries["at"] == 4 + {"cascade": "pull", "task": "enrich", "at": 4}, context=context) + assert out["next"] == "prep" and context.made[1].carries["at"] == 5 out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0}, context=context) @@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles( assert started is not None jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr] key=lambda j: j.id) - assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"] - assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"] + assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \ + ["enrich", "prep", "embed", "score"] + assert [j.kind for j in jobs if j.kind != FINAL] == \ + ["enrich", enrich.KIND, "prep", "embed", "score"] assert web["asked"] == [f"{BASE}/acme/1"] embedded = next(j for j in jobs if j.kind == "embed") assert embedded.result["embedded"] == 1 diff --git a/tests/test_extract.py b/tests/test_extract.py index df8c28c..22cefd5 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -16,6 +16,7 @@ from app.config import Config from app.jobs import extract from app.models import Limits from app.storage import Store +from prepped import upsert def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item: @@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model: def test_a_batch_writes_the_columns_and_stamps_extracted_at( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: uid = posting("1").uid - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer( r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior", @@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at( def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1", place="Berlin", remote="hybrid", salary_min=70000) - store.upsert_role(item) + upsert(store, item) ranked(store, item) stub(monkeypatch, lambda roles: [answer( r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000, @@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced( def test_the_next_upsert_keeps_what_extract_filled( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid") for r in roles]) modeltask.run(extract.TASK, config, store) - store.upsert_role(posting("1")) + upsert(store, posting("1")) role = store.role(posting("1").uid) assert role is not None and (role.country, role.seniority) == ("DE", "mid") @@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled( def test_closed_and_answered_roles_are_not_offered( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: for sid in ("open", "closed", "done"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) ranked(store, posting("open"), posting("closed"), posting("done")) with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' " @@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered( def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) ranked(store, posting("1")) stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")]) @@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read( the boards produced. A role nobody ranks is never sent; a batch dealt a uid finishes it.""" for sid in ("best", "second", "low", "unscored", "gated"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) one = store.save_agent("One", "python backend", limits=Limits(extract_top=1, extract_min=0.5)) two = store.save_agent("Two", "python backend", enabled=False) @@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None: def test_disabled_finds_nothing_and_asks_nobody( store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) model = stub(monkeypatch, lambda roles: []) result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store) @@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody( def test_a_role_read_before_without_a_level_is_read_again_for_it( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1", seniority="senior") - store.upsert_role(item) + upsert(store, item) ranked(store, item) with store.transaction() as conn: conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " @@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text( withtext = posting("text", title="Senior Engineer", seniority="mid") noText = posting("teaser", title="Analyst") noText.body = "" - store.upsert_role(withtext) - store.upsert_role(noText) + upsert(store, withtext) + upsert(store, noText) with store.transaction() as conn: conn.execute("UPDATE roles SET teaser = 'Join our analytics team' " "WHERE source_id = 'teaser'") @@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text( def test_an_answer_with_no_level_is_charged_until_given_up( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: item = posting("1") - store.upsert_role(item) + upsert(store, item) ranked(store, item) stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles]) @@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up( def test_each_agents_best_come_first_then_the_rest(store: Store) -> None: items = [posting(s) for s in ("a1", "a2", "a3", "b1")] for item in items: - store.upsert_role(item) + upsert(store, item) one = store.save_agent("One", "python backend") two = store.save_agent("Two", "python backend") store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""), @@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None: ("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"), ("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"), ("5", "lever", "senior", "")]: - store.upsert_role(posting(sid, board=board, seniority=stated)) + upsert(store, posting(sid, board=board, seniority=stated)) with store.transaction() as conn: conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?", (model, sid)) diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 4e5e9f7..5fab1c2 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed from app.storage import Store from test_places import LINES, ORACLE_TOKENS +from prepped import upsert def gazetteer() -> Gazetteer: @@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None: def test_a_joined_line_is_several_places(store: Store) -> None: - store.upsert_role(posting("1", place="Copenhagen; London; Warsaw")) + upsert(store, posting("1", place="Copenhagen; London; Warsaw")) uid = posting("1").uid assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"), (2, "Warsaw", "Warsaw")] @@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None: def test_the_places_of_the_reader_contract_come_first(store: Store) -> None: - store.upsert_role(posting("1", place="Berlin, Munich", places=[ + upsert(store, posting("1", place="Berlin, Munich", places=[ {"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False}, {"text": "Remote (DE)", "remote": True}])) assert texts(store, posting("1").uid) == [ @@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None: def test_places_without_a_line_make_the_line(store: Store) -> None: - store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None}, + upsert(store, posting("1", places=[{"text": "Lund", "remote": None}, {"text": "Malmö", "remote": None}])) role = store.role(posting("1").uid) assert role is not None @@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None: def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None: - store.upsert_role(posting("1", place="Aarhus")) - store.upsert_role(posting("1")) + upsert(store, posting("1", place="Aarhus")) + upsert(store, posting("1")) assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")] - store.upsert_role(posting("1", place="Odense")) + upsert(store, posting("1", place="Odense")) assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")] def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None: - store.upsert_role(posting("1", place="Aarhus")) + upsert(store, posting("1", place="Aarhus")) uid = posting("1").uid with sqlite3.connect(store.path) as conn: conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) " "VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,)) - store.upsert_role(posting("1", place="Aalborg")) + upsert(store, posting("1", place="Aalborg")) assert [(p.text, p.url) for p in store.role_places(uid)] == [ ("Aalborg", ""), ("Odense", "https://copy")] def test_an_extracted_place_is_resolved(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid store.set_extracted(uid, {"place": "Lund, Sweden"}) assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")] @@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]: "malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund", "zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"} for key, line in lines.items(): - store.upsert_role(posting(key, place=line)) + upsert(store, posting(key, place=line)) return {key: posting(key).uid for key in lines} @@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None: def test_a_country_given_without_a_place_still_counts(store: Store) -> None: - store.upsert_role(posting("1", country="DK")) + upsert(store, posting("1", country="DK")) assert uids(store, "country:DK") == {posting("1").uid} @@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store, def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None: for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")): - store.upsert_role(posting(str(n), place=line)) + upsert(store, posting(str(n), place=line)) assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1), ("jena", "Jena", 1)] assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)] def test_an_alias_links_what_it_names(store: Store) -> None: - store.upsert_role(posting("1", place="Kbh. K")) - store.upsert_role(posting("2", place="Copenhagen")) + upsert(store, posting("1", place="Kbh. K")) + upsert(store, posting("2", place="Copenhagen")) uid = posting("1").uid assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)] assert store.add_place_alias("Kbh", "place:copenhagen") == 1 @@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None: def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None: - store.upsert_role(posting("1", place="Oberkochen; Jena")) - store.upsert_role(posting("2", place="Paderborn, Germany")) + upsert(store, posting("1", place="Oberkochen; Jena")) + upsert(store, posting("2", place="Paderborn, Germany")) oberkochen = store.add_place("Oberkochen", "city", "country:DE") store.add_place("Paderborn", "city", "country:DE") assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")] @@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None: def test_relink_places_resolves_every_role_again(store: Store) -> None: - store.upsert_role(posting("1", place="Oberkochen")) + upsert(store, posting("1", place="Oberkochen")) with sqlite3.connect(store.path) as conn: conn.execute("UPDATE role_places SET place_id = " "(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')") @@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup lines = {} for n, (place, country) in enumerate(LINES): item = posting(f"line{n}", place=place, country=country) - store.upsert_role(item) + upsert(store, item) lines[item.uid] = (place, country) return store, lines diff --git a/tests/test_prep.py b/tests/test_prep.py new file mode 100644 index 0000000..cfa6439 --- /dev/null +++ b/tests/test_prep.py @@ -0,0 +1,358 @@ +"""`app.prep`: the host's words mapped centrally, the stage over the due +roles, the unmapped list, the table fingerprint and migration 22. The host +tables here are fakes; each real table is held to its host by R1's and R2's +tests, and to the vocabularies by the last tests of this file.""" + +from __future__ import annotations + +import importlib +import json +import sqlite3 +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app import cli, models, prep # noqa: E402 +from app.prep import hosts, words # noqa: E402 +from app.prep.country import country, of_host # noqa: E402 +from app.storage import Store # noqa: E402 +from desk.models import Item # noqa: E402 +from plugins.readers import _shared # noqa: E402 + +TABLES: dict[str, Any] = { + "acme": { + "seniority": {"experienced": "", "sr": "senior"}, + "employment": {"regular full time": "full_time", "permanent": ""}, + "remote": {"2": "hybrid", "fully": "remote"}, + }, + "codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}}, + "parts": {"match": {"mode": "substring", + "rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}}, +} + + +# ------------------------------------------------------------------- words + +@pytest.mark.parametrize(("field", "word", "value"), [ + ("seniority", "SR", "senior"), + ("seniority", " Experienced ", ""), # known, names no level + ("seniority", "senior", "senior"), # already the vocabulary + ("seniority", "Grand Poobah", None), # unknown + ("employment", "Regular Full Time", "full_time"), + ("employment", "Permanent", ""), + ("employment", "contract", "contract"), + ("employment", "Zero hours", None), + ("remote", "2", "hybrid"), + ("remote", "On-site", "onsite"), # the shared words + ("remote", "on_site", "onsite"), + ("remote", "IN_OFFICE", "onsite"), + ("remote", "office", "onsite"), + ("remote", "Hybrid", "hybrid"), + ("remote", "unspecified", None), + ("seniority", "", ""), +]) +def test_a_word_goes_through_the_hosts_table_then_the_vocabulary( + field: str, word: str, value: str | None) -> None: + assert words.lookup(field, "acme", word, TABLES) == value + + +def test_another_hosts_table_is_not_used() -> None: + assert words.lookup("seniority", "other", "sr", TABLES) is None + + +def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None: + assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time" + assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time" + assert words.lookup("employment", "codes", "software", TABLES) is None + assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary" + assert words.employment("parts", "permanent_full_time", TABLES) == "full_time" + + +def test_seniority_is_the_stated_level_and_the_titles() -> None: + assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff") + assert words.seniority("acme", "", "Engineer", TABLES) == ("", "") + + +def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None: + assert words.remote("acme", True, (), "", TABLES) == "remote" + assert words.remote("acme", "true", (), "", TABLES) == "remote" + assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote" + # False says nothing the places do not: hosts send it for hybrid roles too. + assert words.remote("acme", False, (), "Berlin", TABLES) == "" + assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid" + assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \ + == "remote" + assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}], + "", TABLES) == "remote" + + +# ----------------------------------------------------------------- country + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def ids(store: Store, *texts: str) -> list[int]: + gaz = store.gazetteer() + return [p for text in texts for p in gaz.resolve(text)] + + +def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None: + gaz = store.gazetteer() + assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK" + assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU" + assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == "" + assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB" + assert country(gaz, ids(store, "Europe")) == "EU" + assert country(gaz, [], "dk") == "DK" + assert country(gaz, ids(store, "Remote"), "Sverige") == "SE" + + +@pytest.mark.parametrize(("stated", "code"), [ + ("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"), + ("", ""), ("Narnia", None)]) +def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str, + code: str | None) -> None: + assert of_host(store.gazetteer(), stated) == code + + +# ------------------------------------------------------------------- stage + +def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme", + **raw: Any) -> Item: + found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}", + title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"board": host, "company": "Acme"} | raw) + found.source = "ats" + return found + + +def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row: + [found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",)) + return found + + +def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None: + store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2", + places=[{"text": "Aarhus", "remote": None}])) + raw = row(store, "1") + assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \ + ("SR", "Regular Full Time", "2") + assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \ + ("", "", "", "") + assert raw["prep_due"] == 1 + assert store.role_places(raw["uid"])[0].place_id is None + + done = prep.run(store, tables=TABLES) + + assert (done.read, done.written, done.remapped) == (1, 1, True) + mapped = row(store, "1") + assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"], + mapped["remote"], mapped["country"], mapped["prep_due"]) == \ + ("senior", "senior", "full_time", "hybrid", "DK", 0) + assert store.role_places(mapped["uid"])[0].name == "Aarhus" + + +def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None: + store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) + store.upsert_role(item("2", seniority="sr")) + prep.run(store, tables=TABLES) + store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}])) + store.upsert_role(item("2", seniority="experienced")) + assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1) + # The place stays resolved when its text did not change. + assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus" + + store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}])) + assert row(store, "1")["prep_due"] == 1 + assert prep.run(store, tables=TABLES).read == 2 + assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == "" + + +def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None: + store.upsert_role(item("1", seniority="sr")) + [read] = store.prep_rows("due") + store.upsert_role(item("1", seniority="experienced")) + store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)]) + assert row(store, "1")["prep_due"] == 2 + + +def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract( + store: Store) -> None: + store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin")) + uid = row(store, "1")["uid"] + with store.transaction() as conn: + conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' " + "WHERE uid = ?", (uid,)) + prep.run(store, tables=TABLES) + mapped = row(store, "1") + assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \ + ("senior", "lead", "remote") + # The host's own statement does overwrite. + store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2")) + prep.run(store, tables=TABLES) + assert row(store, "1")["remote"] == "hybrid" + + +def test_the_company_resolves_through_its_aliases(store: Store) -> None: + store.add_company("Ørsted") + store.upsert_role(item("1", company="ORSTED A/S")) + assert row(store, "1")["company_key"] != "ørsted" + prep.run(store, tables=TABLES) + assert row(store, "1")["company_key"] == "ørsted" + + +def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None: + store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified", + country="Narnia")) + store.upsert_role(item("2", seniority="Grand Poobah", remote=False)) + store.upsert_role(item("3", host="other", seniority="Grand Poobah")) + done = prep.run(store, tables=TABLES) + assert done.unmapped == 4 + listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()} + assert listed == {("seniority", "acme", "grand poobah"): 2, + ("seniority", "other", "grand poobah"): 1, + ("remote", "acme", "unspecified"): 1, + ("country", "acme", "narnia"): 1} + first = {w["word"]: w["first_seen"] for w in store.unmapped()} + + # A word a table learns leaves the list at the next run; the rest keep + # the moment they were first met. + learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}} + prep.run(store, tables=learned) + assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \ + first + assert ("seniority", "acme", "grand poobah") not in \ + {(w["field"], w["host"], w["word"]) for w in store.unmapped()} + assert row(store, "1")["seniority"] == "exec" + + +def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None: + store.upsert_role(item("1", seniority="Distinguished")) + store.upsert_role(item("2", seniority="Distinguished")) + with store.transaction() as conn: + conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'") + assert prep.run(store, tables=TABLES).remapped + assert not prep.run(store, tables=TABLES).remapped + assert store.state[prep.STATE] == prep.fingerprint(TABLES) + + learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}} + done = prep.run(store, tables=learned) + assert (done.remapped, done.read) == (True, 1) + assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "") + # `--remap` maps the closed roles too. + assert prep.run(store, tables=learned, remap=True).read == 2 + assert row(store, "2")["seniority"] == "staff" + + +def test_prep_folds_the_copies(store: Store) -> None: + for n in ("1", "2"): + store.upsert_role(item(n, place="Aarhus")) + assert prep.run(store, tables=TABLES).copies == 1 + assert prep.run(store, tables=TABLES, dupes=False).copies is None + + +# --------------------------------------------------------------- migration + +def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: + path = tmp_path / "old.db" + store = Store(path) + store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True, + country="DK")) + store.upsert_role(item("2")) + store.state[prep.STATE] = "old" + with sqlite3.connect(path) as conn: + conn.execute("DROP INDEX idx_roles_prep_due") + for column in ("seniority_host", "employment_host", "remote_host", "country_host", + "employment", "prep_due"): + conn.execute(f"ALTER TABLE roles DROP COLUMN {column}") + conn.execute("DROP TABLE unmapped") + conn.execute("PRAGMA user_version = 21") + + migrated = Store(path) + + assert migrated.migrations_applied == [22] + one = row(migrated, "1") + assert (one["seniority_host"], one["employment_host"], one["remote_host"], + one["country_host"]) == ("senior", "full_time", "true", "DK") + assert row(migrated, "2")["remote_host"] == "" + assert prep.STATE not in migrated.state + # The first prep maps every open role from those words, the vocabulary + # values the old readers stored as themselves. + prep.run(migrated, tables={}) + one = row(migrated, "1") + assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \ + ("senior", "full_time", "remote", "DK") + + +# ------------------------------------------------------------ CLI and API + +def test_app_prep_runs_and_lists_the_unmapped_words( + tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "config.yaml" + path.write_text(f"database: {tmp_path / 'roledesk.db'}\n") + Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah")) + + assert cli.main(["-c", str(path), "prep"]) == 0 + assert json.loads(capsys.readouterr().out)["read"] == 1 + assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0 + assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"] + assert cli.main(["-c", str(path), "prep", "--remap"]) == 0 + assert json.loads(capsys.readouterr().out)["remapped"] is True + + +def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None: + from fastapi.testclient import TestClient + + from app.config import Config, ServerConfig + from app.main import create_app + + app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + app.state.store.upsert_role(item("1", employment="Zero hours")) + prep.run(app.state.store, tables=TABLES) + with TestClient(app) as client: + found = client.get("/api/prep/unmapped").json() + assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \ + [("employment", "acme", "zero hours", 1)] + + +# ------------------------------------------------------------- the tables + +def test_every_host_table_maps_into_its_vocabulary() -> None: + """Passes with no table at all; R1's and R2's files are held here.""" + for host, tables in hosts.load().items(): + for field, table in tables.items(): + if field == "match": + assert table["mode"] in hosts.MATCH_MODES, host + values = [value for _, value in table["rules"]] + field = "employment" + else: + assert all(key == key.strip().lower() for key in table), (host, field) + values = list(table.values()) + assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field) + + +def test_a_match_table_must_say_how_it_matches(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n') + monkeypatch.setattr(hosts, "__path__", [str(tmp_path)]) + monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) + with pytest.raises(ValueError, match=r"loose\.py has MATCH"): + hosts.load() + (tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n') + monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False) + importlib.invalidate_caches() + assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]} + + +def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None: + assert _shared.EMPLOYMENTS == models.EMPLOYMENTS diff --git a/tests/test_pull.py b/tests/test_pull.py index 8afdd1a..2001dbb 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -24,6 +24,7 @@ from app.models import Application # noqa: E402 from plugins import readers # noqa: E402 from plugins.readers import _runner # noqa: E402 from plugins.ats import ATSSource # noqa: E402 +from app import prep # noqa: E402 from app.jobs import cascade, pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 1, "closed": 1, "copies": 0} + "roles": 1, "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -163,13 +164,14 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, - "roles": 0, "closed": 0, "copies": 0} + "roles": 0, "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} -def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: - """The settle's last step: a board's repeats of one job are one role, - and when the kept one closes the next copy is kept in its place.""" +def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None: + """The prep that follows the settle folds: a board's repeats of one job + are one role, and when the kept one closes the next copy is kept in its + place.""" sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") for job_id in ("1", "2", "3"): @@ -182,8 +184,8 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: class FakeTally: results = ({"roles": 3, "run": "run-1"},) - result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result["copies"] == 2 + pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + assert prep.run(store).copies == 2 [kept] = store.roles_page() assert kept.copies == 2 @@ -191,7 +193,7 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None: store.state["ats:ids:greenhouse:acme"] = json.dumps( [f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone]) result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert (result["closed"], result["copies"]) == (1, 1) + assert (result["closed"], prep.run(store).copies) == (1, 1) [after] = store.roles_page() assert after.uid != kept.uid and after.copies == 1 diff --git a/tests/test_score.py b/tests/test_score.py index 439c25e..54f4f79 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402 from app.jobs import cascade, embed, score # noqa: E402 from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402 from app.storage import Store # noqa: E402 +from prepped import upsert # noqa: E402 NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC) @@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *, title=title, body=body, published_at=published_at, raw={"company": company, **raw}) item.source = "ats" - store.upsert_role(item) + upsert(store, item) return item.uid @@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None: # ---------------------------------------------------------------- cascade -def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( +def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise( store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", @@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise( assert batch is not None jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id) stages = [j.result.get("task") for j in jobs if j.kind == FINAL] - # Enrich found no board with role pages, so the embed and score behind it - # are passed over. - assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"] + # Enrich found no board with role pages, so the prep, embed and score + # behind it are passed over. + assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score", + "summarise"] work = [j.kind for j in jobs if j.kind != FINAL] # The `pull` job and the chain it asked for are one cascade, so the Jobs # tab shows a scheduled pull as one run. Extract and summarise are off and # find nothing; score runs twice, before and after extract. - assert work == ["pull", "pull_batch", "embed", "score", "score"] + assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"] assert all(j.state == "done" for j in jobs) assert len(store.scores(agent.id)) == 1 # The board was stamped with the run the `pull` job claimed, and the @@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None: def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None: """By position, not by name: `score` is in the chain twice, once before extract and once after, and the second must lead on to summarise.""" - assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score", - "extract", "score", "summarise") - assert [cascade.follows("pull", at) for at in range(9)] == \ - ["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""] - assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == "" + assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep", + "embed", "score", "extract", "score", "summarise") + assert [cascade.follows("pull", at) for at in range(11)] == \ + ["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score", + "summarise", ""] + assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == "" assert cascade.follows("nope", 0) == "" assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0} @@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None: made.append(t) return t - out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5}, - context=Context()) - assert out == {"task": "score", "next": "extract"} - assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6} out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7}, context=Context()) - assert out["next"] == "summarise" and made[1].carries["at"] == 8 + assert out == {"task": "score", "next": "extract"} + assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8} + out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9}, + context=Context()) + assert out["next"] == "summarise" and made[1].carries["at"] == 10 diff --git a/tests/test_search.py b/tests/test_search.py index d253985..fdefec1 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -20,6 +20,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 +from app import prep # noqa: E402 from plugins import readers, search # noqa: E402 from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 @@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402 from pipeline import Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 +from prepped import upsert # noqa: E402 + FIXTURES = ROOT / "tests" / "fixtures" / "search" @@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> it.source = "ats" return it - store.upsert_role(item("a", place="Copenhagen, Denmark")) - store.upsert_role(item("b", place="Remote", country="EU")) - store.upsert_role(item("c", place="Somewhere")) + upsert(store, item("a", place="Copenhagen, Denmark")) + upsert(store, item("b", place="Remote", country="EU")) + upsert(store, item("c", place="Somewhere")) assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} @@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo def _pull(store: Store) -> dict[str, Any]: results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] - return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + prep.run(store) + return closed def test_a_complete_search_closes_what_it_stopped_listing( diff --git a/tests/test_storage.py b/tests/test_storage.py index b3fc5c2..2573381 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -19,6 +19,7 @@ from app.models import ( ) from app.storage.store import pack from app.storage import LATEST, Store +from prepped import upsert class Clock: @@ -69,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 21 + assert LATEST == 22 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -186,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -236,9 +237,9 @@ def _downgrade_to_v4(path: Path) -> None: def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("liked")) - store.upsert_role(posting("passed")) - store.upsert_role(posting("none")) + upsert(store, posting("liked")) + upsert(store, posting("passed")) + upsert(store, posting("none")) store.add_company("Acme", key="acme") store.add_company("Other", key="other") # The disabled agent has the lower id; the enabled one is the default. @@ -274,7 +275,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it( tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("liked")) + upsert(store, posting("liked")) store.add_company("Acme", key="acme") _downgrade_to_v4(path) with sqlite3.connect(path) as conn: @@ -292,7 +293,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it( def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None: path = tmp_path / "v4.db" store = Store(path) - store.upsert_role(posting("plain")) + upsert(store, posting("plain")) store.add_company("Acme", key="acme") _downgrade_to_v4(path) @@ -322,8 +323,8 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( old_date = datetime(2020, 1, 1, tzinfo=UTC) trusted = posting("1", board="greenhouse", published_at=old_date) untrusted = posting("2", published_at=old_date) - store.upsert_role(trusted) - store.upsert_role(untrusted) + upsert(store, trusted) + upsert(store, untrusted) before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid) assert before_trusted is not None and before_untrusted is not None @@ -354,7 +355,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( given = posting("3", place="Bogota, CO", country="US") # the board said so extracted = posting("4", place="Bogota, CO") for item in (wrong, several, given, extracted): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v14(conn) conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") @@ -375,7 +376,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None: path = tmp_path / "v13.db" store = Store(path) - store.upsert_role(posting("a")) + upsert(store, posting("a")) uid = posting("a").uid backend = store.save_agent("Backend") blank = store.save_agent("Blank") @@ -391,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -415,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21] + assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -433,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( unknown = posting("unknown") unknown.title = "Backend Engineer" for item in (plain, extracted, kept, unknown): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v17(conn) conn.execute("UPDATE roles SET seniority = ''") @@ -444,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18, 19, 20, 21] + assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -492,8 +493,8 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None: def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage( store: Store) -> None: - store.upsert_role(posting("a")) - store.upsert_role(posting("b")) + upsert(store, posting("a")) + upsert(store, posting("b")) a, b = posting("a").uid, posting("b").uid top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0) blank = store.save_agent("Blank", limits=top, @@ -585,11 +586,11 @@ def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None: def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( store: Store, clock: Clock) -> None: - assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True + assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is True first = store.role(posting().uid) assert first is not None clock.tick() - assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False + assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is False again = store.role(posting().uid) assert again is not None assert again.first_seen == first.first_seen == first.created_at @@ -601,18 +602,18 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None: listed = posting() listed.body = "" - store.upsert_role(listed) + upsert(store, listed) store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) assert store.roles_to_embed("m", 10) == [] - store.upsert_role(listed) # listed again, still without text + upsert(store, listed) # listed again, still without text assert store.roles_to_embed("m", 10) == [] - store.upsert_role(posting()) # the role's own page was read + upsert(store, posting()) # the role's own page was read assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(listed) # later pulls send no text - store.upsert_role(posting()) # or the same text again + upsert(store, listed) # later pulls send no text + upsert(store, posting()) # or the same text again assert store.roles_to_embed("m", 10) == [] @@ -624,9 +625,9 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives( listed.body = "" bare = posting() bare.body = "" - store.upsert_role(bare) # the title alone + upsert(store, bare) # the title alone store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(listed) # a teaser beats the title + upsert(store, listed) # a teaser beats the title assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] role = store.role(listed.uid) assert role is not None and role.teaser == "Build the payments core." @@ -634,21 +635,21 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives( assert store.roles_page()[0].excerpt == "Build the payments core." store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(bare) # a pull without it keeps it + upsert(store, bare) # a pull without it keeps it assert store.roles_to_embed("m", 10) == [] assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr] - store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it + upsert(store, posting(teaser="Build the payments core.")) # the text beats it assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] role = store.role(listed.uid) assert role is not None and document(role).endswith("We build things.") store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) - store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text + upsert(store, posting(teaser="Another teaser.")) # a new teaser under the text assert store.roles_to_embed("m", 10) == [] def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None: - store.upsert_role(posting("1")) # "Senior Backend Engineer" + upsert(store, posting("1")) # "Senior Backend Engineer" uid = posting("1").uid def levels() -> tuple[str, str, str, str]: @@ -658,12 +659,12 @@ def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> No role.seniority_title) assert levels() == ("senior", "", "", "senior") - store.upsert_role(posting("1", seniority="lead")) + upsert(store, posting("1", seniority="lead")) assert levels() == ("lead", "", "lead", "senior") - store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary + upsert(store, posting("1", seniority="Teamleitung")) # not in the vocabulary assert levels() == ("senior", "", "", "senior") store.set_extracted(uid, {"seniority": "staff"}) - store.upsert_role(posting("1", seniority="lead")) + upsert(store, posting("1", seniority="lead")) assert levels() == ("staff", "staff", "lead", "senior") @@ -675,7 +676,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> stated = posting("stated", seniority="lead") extracted = posting("extracted") for item in (joined, detailed, stated, extracted): - store.upsert_role(item) + upsert(store, item) with sqlite3.connect(path) as conn: _undo_v19(conn) # A role whose own page was read is pulled with an empty place in raw. @@ -686,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> migrated = Store(path) - assert migrated.migrations_applied == [19, 20, 21] + assert migrated.migrations_applied == [19, 20, 21, 22] assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] assert migrated.role_places(stated.uid) == [] @@ -703,7 +704,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: - store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000", + upsert(store, posting(place="Berlin", remote=True, salary_min="80000", salary_max=95000, currency="EUR", department="Eng")) role = store.role(posting().uid) assert role is not None @@ -713,7 +714,7 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: assert role.description == "We build things." assert role.original_url.endswith("/123") # A later fetch saying less does not erase what was known. - store.upsert_role(posting()) + upsert(store, posting()) role = store.role(posting().uid) assert role is not None assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000) @@ -725,16 +726,16 @@ def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None: every pull, and a pull that states no pay at all clears it rather than leaving a stale line behind — the opposite of the keep-filled rule `test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above.""" - store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year")) + upsert(store, posting(salary_text="EUR 70,000 to 90,000 per year")) role = store.role(posting().uid) assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year" - store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year")) + upsert(store, posting(salary_text="EUR 75,000 to 95,000 per year")) role = store.role(posting().uid) assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year" # The board stops stating pay: the stored line is cleared, not kept. - store.upsert_role(posting()) + upsert(store, posting()) role = store.role(posting().uid) assert role is not None and role.salary_text == "" @@ -743,10 +744,10 @@ def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) -> """Not a stored column (docs/decisions.md): 'normalised' needs only `salary_min` or `salary_max`, whatever `salary_text` says beside it; 'stated' is `salary_text` alone; neither is ''.""" - store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year", + upsert(store, posting("1", salary_text="EUR 70,000 to 90,000 per year", salary_min=70000, salary_max=90000, currency="EUR")) - store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("3")) + upsert(store, posting("2", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("3")) by_id = {r.source_id: r for r in store.roles()} assert by_id["1"].salary_state == "normalised" assert by_id["2"].salary_state == "stated" @@ -760,27 +761,27 @@ def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest( that finally reports the true, earlier date must still win — an edit on the board only ever moves published_at later, never earlier, so the earliest posted_at ever computed is the one to keep.""" - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 20, tzinfo=UTC))) first = store.role(posting().uid) assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00" clock.tick() - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 1, 1, tzinfo=UTC))) earlier = store.role(posting().uid) assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00" clock.tick() # A later, LATER published_at (an edit) must not push posted_at forward. - store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25, + upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 25, tzinfo=UTC))) still_earlier = store.role(posting().uid) assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00" def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None: - store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC))) + upsert(store, posting(published_at=datetime(2020, 1, 1, tzinfo=UTC))) role = store.role(posting().uid) assert role is not None assert role.posted_at == role.first_seen @@ -791,7 +792,7 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No only set it before extract has run, and once extract has answered — even with a different function than the pull's guess — no later pull may put the guess back. The no-flip property across pull, extract, pull.""" - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) uid = posting().uid assert store.role(uid).function == "engineering" # type: ignore[union-attr] @@ -801,28 +802,28 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No # A later pull with the classifier's old (now wrong) guess must not flip # it back — the "board wins" COALESCE that every other raw field gets # would otherwise put "engineering" straight back on every pull. - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) assert store.role(uid).function == "sales" # type: ignore[union-attr] def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None: - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) uid = posting().uid store.set_extracted(uid, {}) # extract answered this role, but not function assert store.role(uid).extracted_at is not None # type: ignore[union-attr] # extracted_at alone freezes it, whatever a later pull's guess says. - store.upsert_role(posting(function="sales")) + upsert(store, posting(function="sales")) assert store.role(uid).function == "engineering" # type: ignore[union-attr] def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None: - store.upsert_role(posting(function="engineering")) + upsert(store, posting(function="engineering")) role = store.role(posting().uid) assert role is not None and role.function == "engineering" # Before extract, a fresh guess still overwrites the old one, the same # "board wins" behaviour as place, country and the rest. - store.upsert_role(posting(function="sales")) + upsert(store, posting(function="sales")) role = store.role(posting().uid) assert role is not None and role.function == "sales" @@ -834,33 +835,33 @@ def _folded(store: Store, uid: str) -> str: def test_place_fold_is_written_with_place(store: Store) -> None: - store.upsert_role(posting("1", place="København K")) + upsert(store, posting("1", place="København K")) uid = posting("1").uid assert _folded(store, uid) == "kobenhavn k" - store.upsert_role(posting("1")) # says less: both kept + upsert(store, posting("1")) # says less: both kept assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr] "København K", "kobenhavn k") - store.upsert_role(posting("1", place="Kgs. Lyngby")) + upsert(store, posting("1", place="Kgs. Lyngby")) assert _folded(store, uid) == "kgs lyngby" - store.upsert_role(posting("2")) + upsert(store, posting("2")) store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"}) assert _folded(store, posting("2").uid) == "malmo sweden" def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None: - store.upsert_role(posting("1")) # "Senior Backend Engineer" + upsert(store, posting("1")) # "Senior Backend Engineer" uid = posting("1").uid assert store.role(uid).seniority == "senior" # type: ignore[union-attr] retitled = posting("1") retitled.title = "Backend Engineer" - store.upsert_role(retitled) # the title no longer says + upsert(store, retitled) # the title no longer says assert store.role(uid).seniority == "" # type: ignore[union-attr] - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's assert store.role(uid).seniority == "lead" # type: ignore[union-attr] - store.upsert_role(posting("1")) # and a pull does not put it back + upsert(store, posting("1")) # and a pull does not put it back assert store.role(uid).seniority == "lead" # type: ignore[union-attr] store.set_extracted(uid, {}) assert store.role(uid).seniority == "lead" # type: ignore[union-attr] @@ -868,14 +869,14 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both( store: Store) -> None: - store.upsert_role(posting("1", seniority="junior")) + upsert(store, posting("1", seniority="junior")) uid = posting("1").uid role = store.role(uid) assert role is not None assert (role.seniority, role.seniority_stated, role.seniority_title, role.seniority_model) == ("junior", "junior", "senior", "") store.set_extracted(uid, {"seniority": "staff"}) - store.upsert_role(posting("1", seniority="junior")) + upsert(store, posting("1", seniority="junior")) role = store.role(uid) assert role is not None assert (role.seniority, role.seniority_stated, role.seniority_model) == \ @@ -884,18 +885,18 @@ def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both( def test_company_falls_back_to_the_subscription(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse") - store.upsert_role(posting(), sub) + upsert(store, posting(), sub) role = store.role(posting().uid) assert role is not None assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id) def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) clock.tick() - store.upsert_role(posting("2", company="Acme", place="Munich")) + upsert(store, posting("2", company="Acme", place="Munich")) clock.tick() - store.upsert_role(posting("3", company="Other", place="Berlin")) + upsert(store, posting("3", company="Other", place="Berlin")) assert [r.source_id for r in store.roles()] == ["3", "2", "1"] assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"] assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"] @@ -905,9 +906,9 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None: def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None: """A role from a trusted board found today but posted months ago must not outrank one the app has genuinely known about for longer.""" - store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen + upsert(store, posting("recent")) # untrusted: posted_at=first_seen clock.tick() - store.upsert_role(posting("old", board="greenhouse", + upsert(store, posting("old", board="greenhouse", published_at=datetime(2020, 1, 1, tzinfo=UTC))) assert [r.source_id for r in store.roles()] == ["recent", "old"] assert [r.source_id for r in store.roles(days=30)] == ["recent"] @@ -933,7 +934,7 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index( can instead pick `idx_roles_closed` and sort every open role, which on the live database (15,558 of 15,815 roles open) measured over 30x slower than the plan this asserts (see docs/decisions.md).""" - store.upsert_role(posting("1")) + upsert(store, posting("1")) sql, _ = store._roles_from( company=None, place=None, open_only=True, q=None, function=None, remote=None, days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any", @@ -955,10 +956,10 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index( def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin", + upsert(store, posting("1", company="Acme", place="Berlin", function="engineering", remote=True)) clock.tick(hours=48) - store.upsert_role(posting("2", company="Acme", place="Munich", + upsert(store, posting("2", company="Acme", place="Munich", function="sales")) assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"] assert [r.source_id for r in store.roles(q="backend nonsense")] == [] @@ -977,12 +978,12 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No """`pay` filters in SQL, the same as the other filters `_roles_from` builds, so it narrows what a `limit` is taken from rather than the page already built (docs/decisions.md, "filtering before LIMIT").""" - store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000, + upsert(store, posting("normalised", salary_min=70000, salary_max=90000, currency="EUR", salary_text="EUR 70,000 to 90,000 per year")) clock.tick() - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) clock.tick() - store.upsert_role(posting("none")) + upsert(store, posting("none")) assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"} assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"] assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"] @@ -993,9 +994,9 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No def test_roles_filter_by_company_keys(store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme Labs")) - store.upsert_role(posting("3", company="Other")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme Labs")) + upsert(store, posting("3", company="Other")) assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"} assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"} assert store.count_roles(company_keys=["acme-labs"]) == 1 @@ -1008,7 +1009,7 @@ def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None: ("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"), ("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"), ("berlin", "Berlin", "")): - store.upsert_role(posting(sid, place=place, country=country)) + upsert(store, posting(sid, place=place, country=country)) def found(*tokens: str) -> set[str]: return {r.source_id for r in store.roles(places=tokens)} @@ -1030,7 +1031,7 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store ("lead", "Tech Lead"), ("none", "Engineer")): item = posting(sid) item.title = title - store.upsert_role(item) + upsert(store, item) def found(*levels: str, unknown: bool = True) -> set[str]: return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)} @@ -1046,9 +1047,9 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None: - store.upsert_role(posting("normalised", salary_min=70000, currency="EUR")) - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("none")) + upsert(store, posting("normalised", salary_min=70000, currency="EUR")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("none")) assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \ {"normalised", "none"} assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \ @@ -1066,7 +1067,7 @@ def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store) ("e", "Senior Engineer", {"company": "Other"})): item = posting(sid, **raw) item.title = title - store.upsert_role(item) + upsert(store, item) facets = store.role_facets(seniority=["junior"], seniority_unknown=False, pay="normalised") @@ -1087,10 +1088,10 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) # One key, three spellings: the company row's name first, then the roles' # by how many carry each. store.add_company("Ørsted Group", key="ørsted") - store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte")) - store.upsert_role(posting("2", company="Ørsted", place="Gentofte")) - store.upsert_role(posting("3", company="Ørsted", place="Malmö")) - store.upsert_role(posting("4", company="Hidden", place="Malmö")) + upsert(store, posting("1", company="Ørsted A/S", place="Gentofte")) + upsert(store, posting("2", company="Ørsted", place="Gentofte")) + upsert(store, posting("3", company="Ørsted", place="Malmö")) + upsert(store, posting("4", company="Hidden", place="Malmö")) agent = store.save_agent("A") store.set_affinity(agent.id, "hidden", -1) store.set_affinity(agent.id, "ørsted", 2) @@ -1099,17 +1100,17 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store) assert sorted(store.open_companies()) == [ ("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)] assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)] - assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)] + assert sorted(store.open_places()) == [("Gentofte", "DK", 2), ("Malmö", "SE", 2)] def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None: agent = store.save_agent("A") store.add_company("Acme", key="acme") store.set_affinity(agent.id, "acme", -1) - store.upsert_role(posting("other", company="Other")) + upsert(store, posting("other", company="Other")) clock.tick() for i in range(3): - store.upsert_role(posting(f"a{i}", company="Acme")) + upsert(store, posting(f"a{i}", company="Acme")) clock.tick() # Acme's roles are the three newest, so a plain limit=1 shows only Acme; # hide_muted has to reach past all three to find "other" rather than @@ -1128,7 +1129,7 @@ def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) def test_roles_page_by_offset_and_count_the_same_filter( store: Store, clock: Clock) -> None: for sid in ("1", "2", "3", "4"): - store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other")) + upsert(store, posting(sid, company="Acme" if sid != "4" else "Other")) clock.tick() assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"] assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"] @@ -1142,7 +1143,7 @@ def test_roles_page_by_offset_and_count_the_same_filter( def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None: for sid in ("up", "down", "fine", "none"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") role = store.set_vote(agent.id, uid["up"], 1) @@ -1179,7 +1180,7 @@ def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> N def test_votes_are_per_agent(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid one = store.save_agent("One") two = store.save_agent("Two") @@ -1209,7 +1210,7 @@ def test_votes_are_per_agent(store: Store) -> None: def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes( store: Store, clock: Clock) -> None: for sid in ("1", "2", "3"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -1226,7 +1227,7 @@ def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes( def test_read_state_is_set_cleared_and_a_vote_marks_read( store: Store, clock: Clock) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid role = store.role(uid) assert role is not None and role.read_at is None @@ -1252,8 +1253,8 @@ def test_read_state_is_set_cleared_and_a_vote_marks_read( def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid = {r.source_id: r.uid for r in store.roles()} store.set_read(uid["1"]) @@ -1263,7 +1264,7 @@ def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid long_tag = "x" * 50 @@ -1283,9 +1284,9 @@ def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None: def test_roles_filters_by_tags_any_and_all(store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) - store.upsert_role(posting("3")) + upsert(store, posting("1")) + upsert(store, posting("2")) + upsert(store, posting("3")) uid = {r.source_id: r.uid for r in store.roles()} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python", "frontend"]) @@ -1305,9 +1306,9 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) -> store.add_company("Hidden", key="hidden") agent = store.save_agent("A") store.set_affinity(agent.id, "hidden", -1) - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme")) - store.upsert_role(posting("3", company="Hidden")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme")) + upsert(store, posting("3", company="Hidden")) uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python"]) @@ -1320,7 +1321,7 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) -> def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None: - store.upsert_role(posting("a")) + upsert(store, posting("a")) uid = store.roles()[0].uid agent = store.save_agent("A") store.delete_agent(agent.id) @@ -1330,7 +1331,7 @@ def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Sto def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None: for sid in ("a", "b", "c"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") other = store.save_agent("B") @@ -1354,13 +1355,13 @@ def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None sub = store.add_subscription("ats", "greenhouse:acme") other = store.add_subscription("ats", "greenhouse:other") for source_id in ("1", "2", "3"): - store.upsert_role(posting(source_id), sub) - store.upsert_role(posting("9"), other) + upsert(store, posting(source_id), sub) + upsert(store, posting("9"), other) assert store.mark_closed("ats", ["1", "3"], sub.id) == 1 assert {r.source_id for r in store.roles()} == {"1", "3", "9"} assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"} # Listed again: open again. - store.upsert_role(posting("2"), sub) + upsert(store, posting("2"), sub) assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"} @@ -1368,7 +1369,7 @@ def test_an_item_without_a_source_is_refused(store: Store) -> None: item = posting() item.source = "" with pytest.raises(ValueError, match="no source"): - store.upsert_role(item) + upsert(store, item) # --------------------------------------------------------------- companies @@ -1472,7 +1473,7 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None: def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None: for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"), ("kbh", "København K", ""), ("none", "", "")): - store.upsert_role(posting(sid, place=place, country=country)) + upsert(store, posting(sid, place=place, country=country)) agent = store.save_agent("A") uid = {r.source_id: r.uid for r in store.roles()} @@ -1490,8 +1491,8 @@ def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None def test_open_role_counts_and_company_boards(store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme") - store.upsert_role(posting("1", company="Acme", place="Berlin"), sub) - store.upsert_role(posting("2", company="Acme", place="Munich"), sub) + upsert(store, posting("1", company="Acme", place="Berlin"), sub) + upsert(store, posting("2", company="Acme", place="Munich"), sub) store.mark_closed("ats", ["1"], sub.id) assert store.open_role_counts() == {"acme": 1} assert store.company_boards() == {"acme": "greenhouse"} @@ -1524,7 +1525,7 @@ def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) -> store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme") _, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta") assert beta is not None - store.upsert_role(posting("1", company="Beta"), beta) + upsert(store, posting("1", company="Beta"), beta) with pytest.raises(ValueError): store.set_company_board("beta", "greenhouse:acme", host="greenhouse", fields={"ats": "greenhouse"}) @@ -1539,8 +1540,8 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles( name="Acme") assert old is not None store.update_subscription(old.id, fetch=False, target_id="gh-1") - store.upsert_role(posting("1", company="Acme"), old) - store.upsert_role(posting("2", company="Acme"), old) + upsert(store, posting("1", company="Acme"), old) + upsert(store, posting("2", company="Acme"), old) company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby", fields={"ats": "ashby"}) @@ -1550,7 +1551,7 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles( "ashby:acme", "ashby", "", 1, 1) assert company.ats == "ashby" # The same board again closes nothing. - store.upsert_role(posting("3", company="Acme"), new) + upsert(store, posting("3", company="Acme"), new) assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0 assert [r.source_id for r in store.roles()] == ["3"] @@ -1560,7 +1561,7 @@ def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row( _, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme") assert sub is not None - store.upsert_role(posting("1", company="Acme"), sub) + upsert(store, posting("1", company="Acme"), sub) company, detached, closed = store.set_company_board("acme", "") assert closed == 1 and company.key == "acme" assert detached is not None @@ -1627,7 +1628,7 @@ def test_events_are_append_only(store: Store) -> None: def test_an_application_may_point_at_a_role(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) saved = store.save_application(Application(role_uid=posting().uid, status="applied")) assert saved.role_uid == posting().uid assert [e.status for e in store.application_events(saved.id)] == ["applied"] @@ -1636,7 +1637,7 @@ def test_an_application_may_point_at_a_role(store: Store) -> None: # -------------------------------------------------- summary and posting text def test_the_posting_text_is_stored_and_stamped(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) role = store.set_article_text(posting().uid, "The whole posting.") assert role is not None assert role.article_text == "The whole posting." and role.article_at == store.now() @@ -1644,7 +1645,7 @@ def test_the_posting_text_is_stored_and_stamped(store: Store) -> None: def test_a_summary_is_written_once(store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) uid = posting().uid assert store.set_summary(uid, "First.") is True assert store.set_summary(uid, "Second.") is False @@ -1655,7 +1656,7 @@ def test_a_summary_is_written_once(store: Store) -> None: def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer( store: Store) -> None: - store.upsert_role(posting()) + upsert(store, posting()) uid = posting().uid assert store.set_summary(uid, "First.", ["Python", "backend"]) is True role = store.role(uid) @@ -1675,7 +1676,7 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits( agent that hid it. An agent saved before v8 has `limits` of '{}', which reads as the defaults.""" for sid in "abcd": - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) a, b, c, d = (posting(s).uid for s in "abcd") with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d)) diff --git a/tests/test_web.py b/tests/test_web.py index fd5e3d6..9b12e21 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -23,6 +23,7 @@ from app.web.api import _score_after_change from app.web.pages import SECTIONS from auth import AuthConfig from desk.models import Item +from prepped import upsert class Clock: @@ -75,9 +76,9 @@ def client(app: FastAPI) -> Iterator[TestClient]: def test_list_roles_filters_by_query_place_and_function( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering")) - store.upsert_role(posting("2", company="Acme", place="Munich", function="sales")) - store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering")) + upsert(store, posting("1", company="Acme", place="Berlin", function="engineering")) + upsert(store, posting("2", company="Acme", place="Munich", function="sales")) + upsert(store, posting("3", company="Other", place="Berlin", function="engineering")) uid = {r.source_id: r.uid for r in store.roles()} res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"}) @@ -100,7 +101,7 @@ def test_list_roles_filters_by_company_keys_places_and_seniority( ("berlin", "Other", "Berlin", "Senior Engineer")): item = posting(sid, company=company, place=place) item.title = title - store.upsert_role(item) + upsert(store, item) uid = {r.source_id: r.uid for r in store.roles()} def shown(**params: Any) -> set[str]: @@ -133,7 +134,7 @@ def test_role_facets_count_each_facet_under_the_other_filters( ("c", "Engineer", {"salary_text": "EUR 50 per hour"})): item = posting(sid, **raw) item.title = title - store.upsert_role(item) + upsert(store, item) res = client.get("/api/roles/facets", params={"seniority": ["senior"], "seniority_unknown": "false", @@ -152,7 +153,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"), ("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"), ("7", "Metadata Ltd"), ("8", "Hidden Data")): - store.upsert_role(posting(sid, company=company)) + upsert(store, posting(sid, company=company)) agent = store.save_agent("A") store.set_affinity(agent.id, "hidden-data", -1) store.set_affinity(agent.id, "ørsted", 2) @@ -177,7 +178,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None: for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"), ("4", "København K"), ("5", "Kolding")): - store.upsert_role(posting(sid, place=place)) + upsert(store, posting(sid, place=place)) res = client.get("/api/suggest/places", params={"q": "ska"}) assert res.status_code == 200 @@ -200,10 +201,10 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) """`salary_text` and `salary_state` (app/models.py) are on every role the API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before `limit` like the other filters.""" - store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000, + upsert(store, posting("normalised", salary_min=70000, salary_max=90000, currency="EUR", salary_text="EUR 70,000 to 90,000 per year")) - store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour")) - store.upsert_role(posting("none")) + upsert(store, posting("stated", salary_text="USD 45 to 60 per hour")) + upsert(store, posting("none")) uid = {r.source_id: r.uid for r in store.roles()} items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]} @@ -223,7 +224,7 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None: for i in range(3): - store.upsert_role(posting(str(i))) + upsert(store, posting(str(i))) assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1 assert client.get("/api/roles", params={"limit": 0}).status_code == 422 assert client.get("/api/roles", params={"limit": 1001}).status_code == 422 @@ -231,8 +232,8 @@ def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None: def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None: store.add_company("Acme", key="acme") - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Other")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Other")) agent = store.save_agent("A") other = store.save_agent("B") uid = {r.source_id: r.uid for r in store.roles()} @@ -265,10 +266,10 @@ def test_muted_is_excluded_before_the_limit_not_after( store.clock = clock store.add_company("Acme", key="acme") store.set_affinity(store.save_agent("A").id, "acme", -1) - store.upsert_role(posting("other", company="Other")) + upsert(store, posting("other", company="Other")) clock.tick() for i in range(3): - store.upsert_role(posting(f"a{i}", company="Acme")) + upsert(store, posting(f"a{i}", company="Acme")) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} @@ -282,7 +283,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total( clock = Clock() store.clock = clock for i in range(5): - store.upsert_role(posting(str(i))) + upsert(store, posting(str(i))) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} @@ -298,7 +299,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total( def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None: for sid in ("liked", "passed", "fine", "open"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -340,7 +341,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid one = store.save_agent("One") two = store.save_agent("Two") @@ -364,7 +365,7 @@ def test_a_vote_or_an_affinity_needs_an_agent_to_belong_to( client: TestClient, store: Store) -> None: """Nothing to vote as: a 422 with a sentence, not a vote that lands nowhere. Reading is fine, everything is simply unjudged and neutral.""" - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.add_company("Acme", key="acme") uid = posting("1").uid @@ -382,7 +383,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes( clock = Clock() store.clock = clock for sid in ("1", "2", "3"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) clock.tick() uid = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") @@ -403,7 +404,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes( def test_a_vote_moves_a_role_in_the_ranking_at_once( client: TestClient, store: Store) -> None: for sid in ("a", "b", "c"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uid = {r.source_id: r.uid for r in store.roles()} label = {v: k for k, v in uid.items()} agent = store.save_agent("A") @@ -424,7 +425,7 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once( def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid res = client.post(f"/api/roles/{uid}/read") @@ -440,8 +441,8 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) ids = {r.source_id: r.uid for r in store.roles()} uid = posting("1").uid client.post(f"/api/roles/{uid}/read") @@ -455,8 +456,8 @@ def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid = {r.source_id: r.uid for r in store.roles()} store.set_tags(uid["1"], ["python", "backend"]) store.set_tags(uid["2"], ["python", "frontend"]) @@ -476,8 +477,8 @@ def test_filters_lists_tag_counts_and_respects_show_muted( store.add_company("Hidden", key="hidden") agent = store.save_agent("A") # the default agent, whose hidden companies count store.set_affinity(agent.id, "hidden", -1) - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Hidden")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Hidden")) uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)} store.set_tags(uid["1"], ["python"]) store.set_tags(uid["2"], ["python"]) @@ -491,7 +492,7 @@ def test_filters_lists_tag_counts_and_respects_show_muted( def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) uid = posting("1").uid res = client.get(f"/api/roles/{uid}") @@ -524,7 +525,7 @@ def test_companies_lists_board_and_open_role_count( client: TestClient, store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme", ats="greenhouse", country="DK") - store.upsert_role(posting("1", company="Acme"), sub) + upsert(store, posting("1", company="Acme"), sub) store.add_company("Other", key="other", domain="other.example") # no board found store.add_company("Novo", key="novo", ats="taleo", # followed, not readable careers_url="https://novo.example/careers") @@ -561,7 +562,7 @@ def test_judging_a_company_known_only_from_a_search_creates_its_row( for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")): item = posting(f"n{i}", company=name) item.source = "search" - store.upsert_role(item, search) + upsert(store, item, search) key = "novo-nordisk" assert store.company(key) is None @@ -712,7 +713,7 @@ def test_the_watchlist_endpoints_404_for_an_unknown_company(client: TestClient) def test_save_creates_an_application_with_a_saved_event( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme", place="Berlin")) + upsert(store, posting("1", company="Acme", place="Berlin")) uid = posting("1").uid res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"}) @@ -744,7 +745,7 @@ def test_save_an_unknown_role_is_refused(client: TestClient) -> None: def test_set_status_validates_and_writes_an_event( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) + upsert(store, posting("1", company="Acme")) uid = posting("1").uid saved = client.post("/api/applications", json={"role_uid": uid}).json() @@ -761,8 +762,8 @@ def test_set_status_validates_and_writes_an_event( def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1", company="Acme")) - store.upsert_role(posting("2", company="Acme")) + upsert(store, posting("1", company="Acme")) + upsert(store, posting("2", company="Acme")) first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json() second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json() @@ -1069,7 +1070,7 @@ def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence( def test_the_role_endpoint_serves_the_agents_score_and_its_prompt( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid agent = store.save_agent("A", prompts=[]) store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")]) @@ -1088,7 +1089,7 @@ def test_the_role_endpoint_serves_the_agents_score_and_its_prompt( def test_deleting_an_agent_takes_its_votes_and_affinities_with_it( client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) store.add_company("Acme", key="acme") uid = posting("1").uid agent = client.post("/api/agents", json={"name": "A"}).json() @@ -1127,7 +1128,7 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( client: TestClient, store: Store) -> None: for sid in ("low", "high", "gated"): - store.upsert_role(posting(sid)) + upsert(store, posting(sid)) uids = {r.source_id: r.uid for r in store.roles()} agent = store.save_agent("A") store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""), @@ -1152,7 +1153,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid", function="engineering") item.body = long_description - store.upsert_role(item) + upsert(store, item) uid = item.uid row = client.get("/api/roles").json()["items"][0] @@ -1184,7 +1185,7 @@ def test_list_roles_page_stays_small_even_with_long_postings( for n in range(60): item = posting(str(n), company="Acme") item.body = big - store.upsert_role(item) + upsert(store, item) with store.transaction() as conn: conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?", (big, json.dumps({"blob": big}), big)) @@ -1197,7 +1198,7 @@ def test_list_roles_page_stays_small_even_with_long_postings( def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None: - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid store.set_summary(uid, "A short, hand-written summary.") row = client.get("/api/roles").json()["items"][0] @@ -1221,9 +1222,9 @@ def test_sources_lists_searches_and_boards_with_their_countries( client: TestClient, store: Store) -> None: board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") - store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board) - store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board) - store.upsert_role(posting("3", company="Acme", place="Aarhus"), board) + upsert(store, posting("1", company="Acme", place="Copenhagen"), board) + upsert(store, posting("2", company="Acme", place="Boston, MA"), board) + upsert(store, posting("3", company="Acme", place="Aarhus"), board) store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", params={"q": "backend", "area": 1}) @@ -1244,7 +1245,7 @@ def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull( client: TestClient, store: Store, queue: Any) -> None: board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", board="greenhouse") - store.upsert_role(posting("1", company="Acme"), board) + upsert(store, posting("1", company="Acme"), board) off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json() assert off["closed"] == 1 and off["source"]["on"] is False @@ -1345,7 +1346,7 @@ def test_a_companys_board_is_set_replaced_and_detached( body = res.json() assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme") sub_id = body["source"]["id"] - store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + upsert(store, posting("1", company="Acme"), store.subscription(sub_id)) [row] = client.get("/api/companies").json() assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == ( "greenhouse", sub_id, "greenhouse", "acme") @@ -1418,7 +1419,7 @@ def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason( search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") item = posting("1", company="Novo") item.source = "search" - store.upsert_role(item, search) + upsert(store, item, search) client.post("/api/companies/novo/affinity", json={"affinity": 1}) [row] = client.get("/api/companies").json() assert (row["from_search"], row["by_hand"]) == (True, False) @@ -1514,14 +1515,14 @@ def test_a_company_is_deleted_unless_it_is_read( ).json()["source"]["id"] res = client.delete("/api/companies/acme") assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"] - store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id)) + upsert(store, posting("1", company="Acme"), store.subscription(sub_id)) client.delete("/api/companies/acme/board") assert client.delete("/api/companies/acme").status_code == 204 # Open roles hold a company even where no board is on: they would be # orphaned from the list. store.add_company("Beta", key="beta") - store.upsert_role(posting("2", company="Beta")) + upsert(store, posting("2", company="Beta")) res = client.delete("/api/companies/beta") assert res.status_code == 409 and "open roles" in res.json()["detail"] assert store.company("beta") is not None @@ -1562,7 +1563,7 @@ def test_the_posting_text_is_fetched_once( from desk.tools import scrape from desk.tools.article import ArticleContent - store.upsert_role(posting("1")) + upsert(store, posting("1")) uid = posting("1").uid asked: list[str] = [] @@ -1608,8 +1609,8 @@ def test_summarise_queues_one_role_and_the_job_writes_it( monkeypatch.setitem(JOBS, "summarise_batch", replace(JOBS["summarise_batch"], isolate=False, timeout=0.0)) client = TestClient(app) - store.upsert_role(posting("1")) - store.upsert_role(posting("2")) + upsert(store, posting("1")) + upsert(store, posting("2")) uid, bare = posting("1").uid, posting("2").uid with store.transaction() as conn: conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,)) @@ -1656,7 +1657,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( board.body = "The whole posting." search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex") for item in (board, search, posting("alone", company="Other")): - store.upsert_role(item) + upsert(store, item) store.rebuild_dupes() page = client.get("/api/roles").json() @@ -1675,7 +1676,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None: store.add_company("Ørsted") - store.upsert_role(posting("1", company="Orsted")) + upsert(store, posting("1", company="Orsted")) store.add_alias("ørsted", "Dong Energy") for q in ("orsted", "ørsted", "dong"):