From 06ff9f549b1fbf567c69abe4be8161915be610b9 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 13:06:22 +0200 Subject: [PATCH] List unlinked places and unmapped words under Sources, fix them in code with the unmapped skill Settings > Sources > Unmapped shows the place texts no place resolves, with their roles' countries and companies, and the host words no table maps, and copies both as text. Prep fingerprints places.seed(), so a town or spelling added to the code is seeded and every open role is mapped again at the next prep. app prep unlinked lists the places on the command line. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/unmapped/SKILL.md | 48 ++++++++++ app/cli.py | 21 ++++- app/prep/stage.py | 21 +++-- app/static/app.css | 12 +++ app/static/sources.js | 138 ++++++++++++++++++++++------ app/storage/store.py | 30 ++++++ app/templates/settings/sources.html | 19 ++++ app/web/api.py | 18 ++++ app/web/pages.py | 1 + docs/decisions.md | 2 + tests/test_gazetteer.py | 10 ++ tests/test_prep.py | 34 +++++++ tests/test_web.py | 2 +- 13 files changed, 313 insertions(+), 43 deletions(-) create mode 100644 .claude/skills/unmapped/SKILL.md diff --git a/.claude/skills/unmapped/SKILL.md b/.claude/skills/unmapped/SKILL.md new file mode 100644 index 0000000..f92149c --- /dev/null +++ b/.claude/skills/unmapped/SKILL.md @@ -0,0 +1,48 @@ +--- +name: unmapped +description: Fix what prep could not place, from the owner's words ("Überlingen is a German town", "Home Based - Americas is remote", "DEU is Germany") or from the list copied off Settings › Sources › Unmapped. Use when the owner names an unlinked place text or an unmapped host word, pastes that list, or asks what is still unmapped. +--- + +# Unmapped places and host words + +Prep never guesses. A place text the gazetteer does not know stays unlinked, and a host word no table maps stays empty. Both are listed on Settings › Sources › Unmapped and by the CLI: + +```bash +uv run app prep unlinked +uv run app prep unmapped +``` + +`unlinked` prints roles, text, the countries prep gave those roles (`-` for none) and the companies that write it. Only open roles count. A text made only of no-place words (`places._PLACELESS`) is never listed. + +Every fix is a line of code, not a database write, so a new database and the server get it too. Prep fingerprints the host tables and `places.seed()`. After an edit the next prep adds the new places and spellings to the database and maps every open role again, with no pull. + +## Where each fix goes + +| The owner says | Edit | +|---|---| +| A town is in a country ("Überlingen is German") | `_CITIES` in `app/models.py`: add the fold (`places.fold`, e.g. `uberlingen`) to that country's `dict.fromkeys(...)` tuple. | +| A town has another spelling ("Kbh is Copenhagen") | `_ALIAS_GROUPS` in `app/places.py`: add the spelling to the town's group. A new group lists the shown name first, and one of its folds also needs a `_CITIES` entry for its country. | +| A country is written another way ("DEU is Germany") | `_COUNTRY_NAMES` in `app/models.py`: append ` \| DEU` to the country's line. | +| A text names no place ("Fleet Place Office", "Solaris SE" is the company) | `_PLACELESS` in `app/places.py`, with only the words that never name a place. | +| A macro region outside Europe ("Americas", "APAC", "AMER") | Ask whether it should be a place. The default is to add the words to `_PLACELESS`, since the gazetteer has Europe as its only continent. | +| A text means remote or hybrid | `_REMOTE_WORDS` in `app/models.py` already reads "home based" and "remote". Add a phrase only when `remote_of` misses it. | +| A host word for level, contract or remote | That host's table in `app/prep/hosts/.py` (`SENIORITY`, `EMPLOYMENT`, `REMOTE`), keyed by the lowercased word. A host with no module gets a new file. `""` means known but says nothing, which removes it from the list. | +| A country word from a host | `_COUNTRY_NAMES`, as above. | +| Two company names are one company | `Store.add_alias(key, name)`. This is data, not code, since companies are added by hand. | + +Check a fold with `uv run python -c "from app import places; print(places.fold('Überlingen'))"`. A city whose country is unclear is a question for the owner, never a guess. When the owner gives a pattern ("all 1KOMMA5° towns are German"), go through the whole list for it and add each town. + +## Steps + +1. Read the list with the two commands above, or take the owner's words. +2. Make the edits. Keep each dictionary's order and wrapping. +3. Run the tests that hold the dictionaries: + ```bash + uv run pytest -q tests/test_places.py tests/test_gazetteer.py tests/test_prep.py + ``` + For a host table, also run `tests/test_prep_hosts_.py`. +4. Apply it now instead of waiting for the next pull. Then run `unlinked` or `unmapped` again and show the owner what left the list: + ```bash + uv run app prep + ``` +5. Commit on main. When another hunk in the same file is uncommitted and not yours, commit only your hunk. Write it with `git diff` into a patch, cut the patch to your hunk, and apply it with `git apply --cached`. diff --git a/app/cli.py b/app/cli.py index a5deb45..3b09ba6 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,7 +8,7 @@ app agents list|prompt app stages [extract|summarise on|off] app dupes - app prep [unmapped] [--remap] + app prep [unmapped|unlinked] [--remap] app db init `app job` queues the work on a queue of its own and drains it here, so a @@ -162,9 +162,10 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("dupes", help="recompute which open roles are copies of one posting") prep = sub.add_parser( - "prep", help="map the roles' host words and fold the copies, or list the unmapped words") - prep.add_argument("action", nargs="?", choices=["unmapped"], - help="list the host words no table maps, instead of running") + "prep", help="map the roles' host words and fold the copies, or list what is unmapped") + prep.add_argument("action", nargs="?", choices=["unmapped", "unlinked"], + help="list the host words no table maps, or the place texts no " + "place resolves, instead of running") prep.add_argument("--remap", action="store_true", help="map every role again, closed ones included") @@ -679,7 +680,9 @@ def run_dupes(args: argparse.Namespace) -> int: def run_prep(args: argparse.Namespace) -> int: """`app prep`: the prep stage now (`app.prep.run`), printing what it did as JSON; `--remap` maps every role again. `app prep unmapped` lists the - host words no table maps, per field and host, most roles first.""" + host words no table maps, per field and host, most roles first; `app + prep unlinked` the place texts of open roles no place resolves, with the + countries and companies of their roles.""" from app import prep from app.storage import Store @@ -692,6 +695,14 @@ def run_prep(args: argparse.Namespace) -> int: for w in found: print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}") return 0 + if args.action == "unlinked": + linked = store.unlinked_places() + if not linked: + print("every place text is linked") + for p in linked: + where = " ".join(f"{c or '-'}:{n}" for c, n in p["countries"][:3]) + print(f"{p['roles']:6} {p['text'][:40]:40} {where:20} {', '.join(p['companies'])}") + return 0 print(json.dumps(prep.run(store, remap=args.remap)._asdict())) return 0 diff --git a/app/prep/stage.py b/app/prep/stage.py index 5c9dc8c..34c7db6 100644 --- a/app/prep/stage.py +++ b/app/prep/stage.py @@ -4,9 +4,11 @@ A pull stores the host's own words and place texts (`Store.upsert_role`) and marks the role due. `run` maps every due role: its company through `company_aliases`, its places against the gazetteer, its level, contract and remote through the host's tables (`app.prep.words`), and its country -from the places (`app.prep.country`). When the tables changed since the -last run (`fingerprint`, kept in state as `prep:tables`), every open role -is mapped again from its stored words, with no new pull. Then the words no +from the places (`app.prep.country`). When the tables or the gazetteer in +code (`places.seed`) changed since the last run (`fingerprint`, kept in +state as `prep:tables`), the new places and spellings are added to the +database and every open role is mapped again from its stored words, with +no new pull. Then the words no table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the whole database. """ @@ -49,9 +51,10 @@ class Prepped(NamedTuple): def fingerprint(tables: Tables = TABLES) -> str: - """A hash of every host table, the shared remote words and `VERSION`.""" - blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS}, - sort_keys=True) + """A hash of every host table, the shared remote words, the gazetteer the + code describes and `VERSION`.""" + blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS, + "places": repr(places.seed())}, sort_keys=True) return hashlib.sha1(blob.encode()).hexdigest() @@ -114,12 +117,14 @@ def unknown(field: str, host: str, word: str, gaz: places.Gazetteer, def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, dupes: bool = True) -> Prepped: - """Map the due roles, or every open one when the tables changed, or - every role with `remap`; list the unmapped words; fold the copies.""" + """Map the due roles, or every open one when the tables or the + gazetteer changed, or every role with `remap`; list the unmapped words; + fold the copies.""" mark = fingerprint(tables) whole = remap or store.state.get(STATE) != mark rows = store.prep_rows("due") if whole: + store.seed_places() due = {r["uid"] for r in rows} rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due] ids = store.resolve_role_places([r["uid"] for r in rows]) diff --git a/app/static/app.css b/app/static/app.css index 8967784..c4be476 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -1331,6 +1331,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di gap: .4rem; font-size: .8rem; color: var(--dim); } .srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; } +/* A count beside a row's name, as the Unmapped rows show their roles. */ +.srow > .count { margin: 0; white-space: nowrap; } /* A company row's Set board / Change editor: its own line under `.sub`, wrapping like `.sub` does, the boxes styled as `.source-form`'s. */ .crow .board-edit { @@ -1360,6 +1362,16 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di .prune > summary { cursor: pointer; color: var(--dim); font-weight: 600; } .prune[open] > summary { margin-bottom: .75rem; } +/* Where the Unmapped list goes when the clipboard is refused, as news's + `.share-url`: outside a secure context (the server at a LAN address) there + is none, and text has to be on the page to be selectable. */ +.copy-out { + margin: .5rem 0 1rem; padding: .5rem .7rem; max-height: 16rem; overflow: auto; + font-family: var(--mono); font-size: .78rem; white-space: pre-wrap; + overflow-wrap: anywhere; background: var(--bg); border: 1px solid var(--line); + border-radius: .3rem; +} + /* applications and companies */ table.list { width: 100%; border-collapse: collapse; background: var(--panel); } diff --git a/app/static/sources.js b/app/static/sources.js index e306ae5..42f1b63 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -19,9 +19,14 @@ * Under the Follow form of Company boards, a fold lists what each reader * declares (`/api/readers`): how its list arrives, where the text is, what * it filters at the source, how it states places, and the robots.txt - * decision taken when it was built. A second fold lists the host words no - * table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and - * host, most roles first; it is left out when there are none. + * decision taken when it was built. + * + * The Unmapped section lists what prep could not place, most roles first: + * the place texts no place resolves (`/api/prep/unlinked`), with the + * countries and companies of their roles, and the host words no table in + * `app/prep/hosts` maps (`/api/prep/unmapped`), per field and host. Copy + * puts both on the clipboard as text for Claude, or, where there is no + * clipboard, on the page selected. */ (function () { "use strict"; @@ -322,36 +327,108 @@ $("#board-form").after(fold); } - /** The host words prep could not map, one row per field and host. */ - function renderUnmapped(words) { - if (!words.length) return; + // ------------------------------------------------------------ unmapped + + /** Rows shown before Show all; the copy always holds every one. */ + const UNLINKED_SHOWN = 40; + /** @type {{places: object[], words: object[]}} */ + const unmapped = { places: [], words: [] }; + + /** "DE 5 · EU 5 · none 1", the three biggest. */ + function where(place) { + const shown = place.countries.slice(0, 3).map(([code, n]) => `${code || "none"} ${n}`); + if (place.countries.length > 3) shown.push("…"); + return shown.join(" · "); + } + + function srow(name, right, sub) { + const row = document.createElement("div"); + row.className = "srow"; + const label = document.createElement("span"); + label.className = "name"; + label.textContent = name; + const n = document.createElement("span"); + n.className = "count"; + n.textContent = right; + const line = document.createElement("span"); + line.className = "sub"; + line.textContent = sub; + row.append(label, n, line); + return row; + } + + function renderUnlinked(all) { + const list = $("#unlinked-list"); + const shown = all ? unmapped.places : unmapped.places.slice(0, UNLINKED_SHOWN); + list.replaceChildren(...shown.map((p) => srow(p.text, count(p.roles, "role"), + [where(p), p.companies.join(", ")].filter(Boolean).join(" · ")))); + $("#unlinked-count").textContent = unmapped.places.length + ? `${count(unmapped.places.length, "place text")} on open roles no place resolves.` + : "Every place text of an open role resolves."; + if (shown.length < unmapped.places.length) { + const more = document.createElement("button"); + more.type = "button"; + more.className = "linky"; + more.textContent = `Show all ${unmapped.places.length}`; + more.addEventListener("click", () => renderUnlinked(true)); + list.appendChild(more); + } + } + + function renderWords() { const groups = new Map(); - for (const w of words) { + for (const w of unmapped.words) { const key = `${w.field} · ${w.host || "no host"}`; if (!groups.has(key)) groups.set(key, []); groups.get(key).push(w); } - const fold = document.createElement("details"); - fold.className = "prune"; - fold.id = "unmapped"; - const summary = document.createElement("summary"); - summary.textContent = `Host words no table maps (${words.length})`; - const list = document.createElement("div"); - list.className = "srows"; - for (const [key, found] of groups) { - const row = document.createElement("div"); - row.className = "srow"; - const name = document.createElement("span"); - name.className = "name"; - name.textContent = key; - const sub = document.createElement("span"); - sub.className = "sub"; - sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · "); - row.append(name, sub); - list.appendChild(row); + $("#words-list").replaceChildren(...[...groups].map(([key, found]) => + srow(key, count(found.reduce((n, w) => n + w.roles, 0), "role"), + found.map((w) => `${w.word} ${w.roles}`).join(" · ")))); + $("#words-count").textContent = unmapped.words.length + ? `${count(unmapped.words.length, "host word")} no table maps.` + : "Every host word is mapped."; + } + + /** Both lists as text, one line an entry, for pasting to Claude. */ + function unmappedText() { + const lines = ["Unlinked places (roles, their countries, companies):"]; + for (const p of unmapped.places) { + lines.push(`${p.roles} ${p.text} [${where(p)}] ${p.companies.join(", ")}`); } - fold.append(summary, list); - ($("#readers") || $("#board-form")).after(fold); + lines.push("", "Unmapped host words (field, host, roles, word):"); + for (const w of unmapped.words) { + lines.push(`${w.field} ${w.host || "-"} ${w.roles} ${w.word}`); + } + return lines.join("\n"); + } + + // The clipboard needs a secure context; the server reached at a LAN + // address is not one. As news's copyLink, the text is then put on the + // page and selected, for the device's own Copy. + async function copyUnmapped() { + const text = unmappedText(); + try { + await navigator.clipboard.writeText(text); + window.toast.good("Copied the unmapped list."); + return; + } catch { /* no clipboard here: show the text instead */ } + const holder = $("#unmapped-text"); + holder.textContent = text; + holder.hidden = false; + const range = document.createRange(); + range.selectNodeContents(holder); + const selection = window.getSelection(); + selection.removeAllRanges(); + selection.addRange(range); + window.toast.info("The list is selected: use your device's Copy."); + } + + async function loadUnmapped() { + [unmapped.places, unmapped.words] = await Promise.all( + [api("/api/prep/unlinked"), api("/api/prep/unmapped")]); + renderUnlinked(false); + renderWords(); } // ------------------------------------------------------ one section shown @@ -395,8 +472,11 @@ document.addEventListener("DOMContentLoaded", async () => { if (!$("#searches-list")) return; if ($("#board-form")) { - api("/api/readers").then(renderReaders, () => {}) - .then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {}); + api("/api/readers").then(renderReaders, () => {}); + } + if ($("#unlinked-list")) { + loadUnmapped().catch(() => {}); + $("#unmapped-copy").addEventListener("click", copyUnmapped); } try { await load(); diff --git a/app/storage/store.py b/app/storage/store.py index eec28c6..1e15076 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -10,6 +10,7 @@ from __future__ import annotations import json import sqlite3 +from collections import Counter from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence from contextlib import closing, contextmanager from datetime import UTC, datetime, timedelta @@ -601,6 +602,35 @@ class Store: gazetteer.relink(conn, gaz, folded) return place_id + def unlinked_places(self, limit: int | None = None) -> list[dict[str, Any]]: + """`unresolved_places` with what places each text: `countries`, the + `[code, roles]` prep gave its roles ('' for none), and `companies`, + the three that write it most, both most roles first. What the Sources + page's Unmapped section and `app prep unlinked` list.""" + found = self.unresolved_places(limit=limit) + countries: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found} + companies: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found} + for r in self._rows( + """SELECT rp.fold, r.country, r.company, COUNT(DISTINCT r.uid) AS n + FROM role_places rp JOIN roles r ON r.uid = rp.uid + WHERE rp.place_id IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL + GROUP BY rp.fold, r.country, r.company"""): + if (fold := str(r["fold"])) in countries: + countries[fold][str(r["country"])] += int(r["n"]) + companies[fold][str(r["company"])] += int(r["n"]) + return [{"fold": fold, "text": text, "roles": n, + "countries": [list(c) for c in countries[fold].most_common()], + "companies": [c for c, _ in companies[fold].most_common(3) if c]} + for fold, text, n in found] + + def seed_places(self) -> int: + """Add what `places.seed` describes and the database lacks, a place + or a spelling written into the code since the file was made. Prep + calls it when the code changed and resolves the open roles again. + Returns how many places were added.""" + with self.transaction() as conn: + return gazetteer.seed(conn) + def relink_places(self) -> int: """Resolve every role's own places again, after the gazetteer was seeded anew. Returns how many roles now resolve differently.""" diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html index 300d65d..a2f9c5a 100644 --- a/app/templates/settings/sources.html +++ b/app/templates/settings/sources.html @@ -163,6 +163,25 @@
loading…
+ +{# What prep could not place, from `/api/prep/unlinked` and + `/api/prep/unmapped`; sources.js draws both lists. A fix is a line of code + (the `unmapped` skill in .claude/skills says where), so the page only + lists, and Copy hands the whole list over as text. #} +
+
+

Unmapped

+ +
+

What prep could not place. A place text no city or country resolves leaves its role without that place, and a host word no table maps leaves the level, contract or remote empty. Tell Claude what one is, such as "Überlingen is a town in Germany" or "Home Based - Americas is remote", or paste the copied list. The fix goes into the code, and the next pull applies it to every open role.

+ +

Places

+

+
+

Host words

+

+
+
{% endblock %} {% block scripts %} diff --git a/app/web/api.py b/app/web/api.py index 724f2b3..2235697 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -932,6 +932,24 @@ def list_unmapped(store: StoreDep) -> list[UnmappedOut]: return [UnmappedOut(**w) for w in store.unmapped()] +class UnlinkedOut(BaseModel): + """A place text the gazetteer does not know (`Store.unlinked_places`).""" + + fold: str + text: str + roles: int + countries: list[tuple[str, int]] + companies: list[str] + + +@router.get("/prep/unlinked") +def list_unlinked(store: StoreDep) -> list[UnlinkedOut]: + """The place texts of open roles no place resolves, most roles first: + what a city, a spelling or a no-place word in `app/models.py` and + `app/places.py` should learn.""" + return [UnlinkedOut(**p) for p in store.unlinked_places()] + + class SearchBody(BaseModel): provider: str name: str = "" diff --git a/app/web/pages.py b/app/web/pages.py index b4f9da5..c8c8583 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -86,6 +86,7 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = { ("job-boards", "Job boards"), ("company-boards", "Company boards"), ("runs", "Runs"), + ("unmapped", "Unmapped"), ), } diff --git a/docs/decisions.md b/docs/decisions.md index 61d39be..a619e81 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -122,6 +122,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Readers scrape; `prep` decides.** A reader hands on the host's own words (`seniority`, `employment`, `remote`, and `country` only when the host states one per role) and does light cleanup only: text from HTML, dates, the places of a role as a list, pay figures. Mapping lived in 25 readers, each spelling its tables its own way, so changing what "Experienced" means touched a plugin. The `prep` stage (`app/prep/`) maps centrally after every pull and every enrich run. Seniority, employment and remote go through the host's table in `app/prep/hosts/.py` (whole words, or `MATCH` fragments by substring or token), then a shared word map. Country comes from the resolved places (several European countries give `EU`), and the host's country is only a fallback. Places and company aliases are resolved, and copies and duplicates are folded over the whole database. `upsert_role` stores the host words (`*_host` columns, migration 22) and marks the role `prep_due`. When a table changes, its fingerprint makes the next prep map every open role again from the stored words, with no pull. A word no table knows is listed in `unmapped` (`app prep unmapped`, the Sources page). A host's `remote: False` stays unknown rather than onsite, because SmartRecruiters, Ashby, Recruitee and Workable send it for hybrid roles too. +**Unmapped places and words are fixed in code, from a list on Sources.** Settings › Sources › Unmapped lists the place texts of open roles no place resolves (`app prep unlinked`, with the countries and companies of their roles) and the host words no table maps. It has no editor: the owner tells Claude what a text is, and the `unmapped` skill in `.claude/skills` says which dictionary or host table takes it. Code rather than `add_place_alias` rows, because a fix then reaches a new database and is reviewed like any other change. The prep fingerprint covers `places.seed()`, so the next prep adds the new places and spellings to the database and maps every open role again. Copy the list puts both lists on the clipboard, or selects them on the page where there is no clipboard. + **Fetch terms narrow what big boards send across Europe.** An agent's `fetch_terms` (migration 23) are words a big board must match when it is read with the wide reach: a company set to All countries, or a one-country host whose reach was dropped for needing over 8 requests. A company set to **All roles** (`companies.all_roles`, migration 24) takes no terms on any of its boards, and will take no category or level filter either. It replaced a board's `{"terms": false}` param, which migration 24 moved to the board's company. On its own, All roles changes nothing today, since the narrow reach takes no terms; it is kept for the role filters to come. The terms of a pull are the union over the enabled agents; one enabled agent without terms switches them off. The narrow reach takes no terms, because it already leaves at most about 150 roles on any big board. The survey of 2026-09-30 found that every host's keyword matches the whole text, so a term cuts less than a title filter would ("engineer" left 1,713 of Bosch's 4,830). SmartRecruiters, SuccessFactors, Workday, Phenom, Eightfold and Oracle take one term a request, with one series per term unioned by id; over 8 series the place filter goes first, then the terms. A term-filtered read is never the whole board: it writes no size, prunes no per-role state and closes nothing. Category and level filters wait until something maps agent terms to host ids. A town in a gate is still fetched as its country. **Seniority has three sources.** `seniority_stated` is the host's word mapped by that host's table in `app/prep/hosts/`, `seniority_title` is `seniority.from_title`, and `seniority_model` is `extract`'s verdict; `roles.seniority` is the first of model, stated, title. `extract` reads roles without a model value, every agent's first pick before anyone's second, with the other two as hints it may overrule, since host vocabularies differ and 57% of roles state nothing. `app seniority` shows per host how often the stated level agrees with the model. diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 5fab1c2..856eb7d 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -319,6 +319,16 @@ def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None: assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)] +def test_unlinked_texts_carry_their_roles_countries_and_companies(store: Store) -> None: + upsert(store, posting("1", place="Oberkochen", company="Zeiss", country="DE")) + upsert(store, posting("2", place="Oberkochen", company="Zeiss")) + upsert(store, posting("3", place="Oberkochen; Remote", company="Hensoldt", country="DE")) + upsert(store, posting("4", place="Remote")) + assert store.unlinked_places() == [{ + "fold": "oberkochen", "text": "Oberkochen", "roles": 3, + "countries": [["DE", 2], ["", 1]], "companies": ["Zeiss", "Hensoldt"]}] + + def test_an_alias_links_what_it_names(store: Store) -> None: upsert(store, posting("1", place="Kbh. K")) upsert(store, posting("2", place="Copenhagen")) diff --git a/tests/test_prep.py b/tests/test_prep.py index d60aec6..b5ef2e0 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -273,6 +273,21 @@ def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) assert row(store, "2")["seniority"] == "staff" +def test_a_town_added_to_the_code_is_linked_at_the_next_prep( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.upsert_role(item("1", places=[{"text": "Überlingen", "remote": None}])) + prep.run(store, tables=TABLES) + assert [p["text"] for p in store.unlinked_places()] == ["Überlingen"] + assert row(store, "1")["country"] == "" + + monkeypatch.setitem(models._CITIES, "uberlingen", "DE") + done = prep.run(store, tables=TABLES) + assert done.remapped + assert store.unlinked_places() == [] + assert row(store, "1")["country"] == "DE" + assert store.state[prep.STATE] == prep.fingerprint(TABLES) + + def test_prep_folds_the_copies(store: Store) -> None: for n in ("1", "2"): store.upsert_role(item(n, place="Aarhus")) @@ -325,6 +340,9 @@ def test_app_prep_runs_and_lists_the_unmapped_words( assert json.loads(capsys.readouterr().out)["read"] == 1 assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0 assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"] + Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Überlingen")) + assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0 + assert capsys.readouterr().out.split() == ["1", "Überlingen", "-:1", "Acme"] assert cli.main(["-c", str(path), "prep", "--remap"]) == 0 assert json.loads(capsys.readouterr().out)["remapped"] is True @@ -345,6 +363,22 @@ def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None: [("employment", "acme", "zero hours", 1)] +def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None: + from fastapi.testclient import TestClient + + from app.config import Config, ServerConfig + from app.main import create_app + + app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + app.state.store.upsert_role(item("1", place="Überlingen", country="DE")) + prep.run(app.state.store, tables=TABLES) + with TestClient(app) as client: + found = client.get("/api/prep/unlinked").json() + assert found == [{"fold": "uberlingen", "text": "Überlingen", "roles": 1, + "countries": [["DE", 1]], "companies": ["Acme"]}] + + # ------------------------------------------------------------- the tables def test_every_host_table_maps_into_its_vocabulary() -> None: diff --git a/tests/test_web.py b/tests/test_web.py index 81f266c..88f3788 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -832,7 +832,7 @@ def test_every_section_of_the_sources_page_exists(client: TestClient) -> None: data-section>` on the page, which is what sources.js shows one of, and the log at the end of Runs still answers `#messages`.""" anchors = [anchor for anchor, _ in SECTIONS["/settings/sources"]] - assert anchors == ["job-boards", "company-boards", "runs"] + assert anchors == ["job-boards", "company-boards", "runs", "unmapped"] page = client.get("/settings/sources").text assert page.count("data-section>") == len(anchors) for anchor in anchors: