List unlinked places and unmapped words under Sources, fix them in code with the unmapped skill

Settings > Sources > Unmapped shows the place texts no place resolves, with
their roles' countries and companies, and the host words no table maps, and
copies both as text. Prep fingerprints places.seed(), so a town or spelling
added to the code is seeded and every open role is mapped again at the next
prep. app prep unlinked lists the places on the command line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:06:22 +02:00
1 parent ae978594b4
commit 06ff9f549b
13 files changed
+313 -43

No files matched your search

+16 -5
View File
@@ -8,7 +8,7 @@
app agents list|prompt
app stages [extract|summarise on|off]
app dupes
app prep [unmapped] [--remap]
app prep [unmapped|unlinked] [--remap]
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -162,9 +162,10 @@ def build_parser() -> argparse.ArgumentParser:
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
prep = sub.add_parser(
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
prep.add_argument("action", nargs="?", choices=["unmapped"],
help="list the host words no table maps, instead of running")
"prep", help="map the roles' host words and fold the copies, or list what is unmapped")
prep.add_argument("action", nargs="?", choices=["unmapped", "unlinked"],
help="list the host words no table maps, or the place texts no "
"place resolves, instead of running")
prep.add_argument("--remap", action="store_true",
help="map every role again, closed ones included")
@@ -679,7 +680,9 @@ def run_dupes(args: argparse.Namespace) -> int:
def run_prep(args: argparse.Namespace) -> int:
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
host words no table maps, per field and host, most roles first."""
host words no table maps, per field and host, most roles first; `app
prep unlinked` the place texts of open roles no place resolves, with the
countries and companies of their roles."""
from app import prep
from app.storage import Store
@@ -692,6 +695,14 @@ def run_prep(args: argparse.Namespace) -> int:
for w in found:
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
return 0
if args.action == "unlinked":
linked = store.unlinked_places()
if not linked:
print("every place text is linked")
for p in linked:
where = " ".join(f"{c or '-'}:{n}" for c, n in p["countries"][:3])
print(f"{p['roles']:6} {p['text'][:40]:40} {where:20} {', '.join(p['companies'])}")
return 0
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
return 0
+13 -8
View File
@@ -4,9 +4,11 @@ A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), and its country
from the places (`app.prep.country`). When the tables changed since the
last run (`fingerprint`, kept in state as `prep:tables`), every open role
is mapped again from its stored words, with no new pull. Then the words no
from the places (`app.prep.country`). When the tables or the gazetteer in
code (`places.seed`) changed since the last run (`fingerprint`, kept in
state as `prep:tables`), the new places and spellings are added to the
database and every open role is mapped again from its stored words, with
no new pull. Then the words no
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
copies across the whole database.
"""
@@ -49,9 +51,10 @@ class Prepped(NamedTuple):
def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
sort_keys=True)
"""A hash of every host table, the shared remote words, the gazetteer the
code describes and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
"places": repr(places.seed())}, sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
@@ -114,12 +117,14 @@ def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
dupes: bool = True) -> Prepped:
"""Map the due roles, or every open one when the tables changed, or
every role with `remap`; list the unmapped words; fold the copies."""
"""Map the due roles, or every open one when the tables or the
gazetteer changed, or every role with `remap`; list the unmapped words;
fold the copies."""
mark = fingerprint(tables)
whole = remap or store.state.get(STATE) != mark
rows = store.prep_rows("due")
if whole:
store.seed_places()
due = {r["uid"] for r in rows}
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
ids = store.resolve_role_places([r["uid"] for r in rows])
+12
View File
@@ -1331,6 +1331,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
gap: .4rem; font-size: .8rem; color: var(--dim);
}
.srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; }
/* A count beside a row's name, as the Unmapped rows show their roles. */
.srow > .count { margin: 0; white-space: nowrap; }
/* A company row's Set board / Change editor: its own line under `.sub`,
wrapping like `.sub` does, the boxes styled as `.source-form`'s. */
.crow .board-edit {
@@ -1360,6 +1362,16 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
.prune > summary { cursor: pointer; color: var(--dim); font-weight: 600; }
.prune[open] > summary { margin-bottom: .75rem; }
/* Where the Unmapped list goes when the clipboard is refused, as news's
`.share-url`: outside a secure context (the server at a LAN address) there
is none, and text has to be on the page to be selectable. */
.copy-out {
margin: .5rem 0 1rem; padding: .5rem .7rem; max-height: 16rem; overflow: auto;
font-family: var(--mono); font-size: .78rem; white-space: pre-wrap;
overflow-wrap: anywhere; background: var(--bg); border: 1px solid var(--line);
border-radius: .3rem;
}
/* applications and companies */
table.list { width: 100%; border-collapse: collapse; background: var(--panel); }
+109 -29
View File
@@ -19,9 +19,14 @@
* Under the Follow form of Company boards, a fold lists what each reader
* declares (`/api/readers`): how its list arrives, where the text is, what
* it filters at the source, how it states places, and the robots.txt
* decision taken when it was built. A second fold lists the host words no
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
* host, most roles first; it is left out when there are none.
* decision taken when it was built.
*
* The Unmapped section lists what prep could not place, most roles first:
* the place texts no place resolves (`/api/prep/unlinked`), with the
* countries and companies of their roles, and the host words no table in
* `app/prep/hosts` maps (`/api/prep/unmapped`), per field and host. Copy
* puts both on the clipboard as text for Claude, or, where there is no
* clipboard, on the page selected.
*/
(function () {
"use strict";
@@ -322,36 +327,108 @@
$("#board-form").after(fold);
}
/** The host words prep could not map, one row per field and host. */
function renderUnmapped(words) {
if (!words.length) return;
// ------------------------------------------------------------ unmapped
/** Rows shown before Show all; the copy always holds every one. */
const UNLINKED_SHOWN = 40;
/** @type {{places: object[], words: object[]}} */
const unmapped = { places: [], words: [] };
/** "DE 5 · EU 5 · none 1", the three biggest. */
function where(place) {
const shown = place.countries.slice(0, 3).map(([code, n]) => `${code || "none"} ${n}`);
if (place.countries.length > 3) shown.push("…");
return shown.join(" · ");
}
function srow(name, right, sub) {
const row = document.createElement("div");
row.className = "srow";
const label = document.createElement("span");
label.className = "name";
label.textContent = name;
const n = document.createElement("span");
n.className = "count";
n.textContent = right;
const line = document.createElement("span");
line.className = "sub";
line.textContent = sub;
row.append(label, n, line);
return row;
}
function renderUnlinked(all) {
const list = $("#unlinked-list");
const shown = all ? unmapped.places : unmapped.places.slice(0, UNLINKED_SHOWN);
list.replaceChildren(...shown.map((p) => srow(p.text, count(p.roles, "role"),
[where(p), p.companies.join(", ")].filter(Boolean).join(" · "))));
$("#unlinked-count").textContent = unmapped.places.length
? `${count(unmapped.places.length, "place text")} on open roles no place resolves.`
: "Every place text of an open role resolves.";
if (shown.length < unmapped.places.length) {
const more = document.createElement("button");
more.type = "button";
more.className = "linky";
more.textContent = `Show all ${unmapped.places.length}`;
more.addEventListener("click", () => renderUnlinked(true));
list.appendChild(more);
}
}
function renderWords() {
const groups = new Map();
for (const w of words) {
for (const w of unmapped.words) {
const key = `${w.field} · ${w.host || "no host"}`;
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(w);
}
const fold = document.createElement("details");
fold.className = "prune";
fold.id = "unmapped";
const summary = document.createElement("summary");
summary.textContent = `Host words no table maps (${words.length})`;
const list = document.createElement("div");
list.className = "srows";
for (const [key, found] of groups) {
const row = document.createElement("div");
row.className = "srow";
const name = document.createElement("span");
name.className = "name";
name.textContent = key;
const sub = document.createElement("span");
sub.className = "sub";
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
row.append(name, sub);
list.appendChild(row);
$("#words-list").replaceChildren(...[...groups].map(([key, found]) =>
srow(key, count(found.reduce((n, w) => n + w.roles, 0), "role"),
found.map((w) => `${w.word} ${w.roles}`).join(" · "))));
$("#words-count").textContent = unmapped.words.length
? `${count(unmapped.words.length, "host word")} no table maps.`
: "Every host word is mapped.";
}
/** Both lists as text, one line an entry, for pasting to Claude. */
function unmappedText() {
const lines = ["Unlinked places (roles, their countries, companies):"];
for (const p of unmapped.places) {
lines.push(`${p.roles} ${p.text} [${where(p)}] ${p.companies.join(", ")}`);
}
fold.append(summary, list);
($("#readers") || $("#board-form")).after(fold);
lines.push("", "Unmapped host words (field, host, roles, word):");
for (const w of unmapped.words) {
lines.push(`${w.field} ${w.host || "-"} ${w.roles} ${w.word}`);
}
return lines.join("\n");
}
// The clipboard needs a secure context; the server reached at a LAN
// address is not one. As news's copyLink, the text is then put on the
// page and selected, for the device's own Copy.
async function copyUnmapped() {
const text = unmappedText();
try {
await navigator.clipboard.writeText(text);
window.toast.good("Copied the unmapped list.");
return;
} catch { /* no clipboard here: show the text instead */ }
const holder = $("#unmapped-text");
holder.textContent = text;
holder.hidden = false;
const range = document.createRange();
range.selectNodeContents(holder);
const selection = window.getSelection();
selection.removeAllRanges();
selection.addRange(range);
window.toast.info("The list is selected: use your device's Copy.");
}
async function loadUnmapped() {
[unmapped.places, unmapped.words] = await Promise.all(
[api("/api/prep/unlinked"), api("/api/prep/unmapped")]);
renderUnlinked(false);
renderWords();
}
// ------------------------------------------------------ one section shown
@@ -395,8 +472,11 @@
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
if ($("#board-form")) {
api("/api/readers").then(renderReaders, () => {})
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
api("/api/readers").then(renderReaders, () => {});
}
if ($("#unlinked-list")) {
loadUnmapped().catch(() => {});
$("#unmapped-copy").addEventListener("click", copyUnmapped);
}
try {
await load();
+30
View File
@@ -10,6 +10,7 @@ from __future__ import annotations
import json
import sqlite3
from collections import Counter
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
@@ -601,6 +602,35 @@ class Store:
gazetteer.relink(conn, gaz, folded)
return place_id
def unlinked_places(self, limit: int | None = None) -> list[dict[str, Any]]:
"""`unresolved_places` with what places each text: `countries`, the
`[code, roles]` prep gave its roles ('' for none), and `companies`,
the three that write it most, both most roles first. What the Sources
page's Unmapped section and `app prep unlinked` list."""
found = self.unresolved_places(limit=limit)
countries: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
companies: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
for r in self._rows(
"""SELECT rp.fold, r.country, r.company, COUNT(DISTINCT r.uid) AS n
FROM role_places rp JOIN roles r ON r.uid = rp.uid
WHERE rp.place_id IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
GROUP BY rp.fold, r.country, r.company"""):
if (fold := str(r["fold"])) in countries:
countries[fold][str(r["country"])] += int(r["n"])
companies[fold][str(r["company"])] += int(r["n"])
return [{"fold": fold, "text": text, "roles": n,
"countries": [list(c) for c in countries[fold].most_common()],
"companies": [c for c, _ in companies[fold].most_common(3) if c]}
for fold, text, n in found]
def seed_places(self) -> int:
"""Add what `places.seed` describes and the database lacks, a place
or a spelling written into the code since the file was made. Prep
calls it when the code changed and resolves the open roles again.
Returns how many places were added."""
with self.transaction() as conn:
return gazetteer.seed(conn)
def relink_places(self) -> int:
"""Resolve every role's own places again, after the gazetteer was
seeded anew. Returns how many roles now resolve differently."""
+19
View File
@@ -163,6 +163,25 @@
<div id="notices"><span class="muted">loading…</span></div>
</div>
</section>
{# What prep could not place, from `/api/prep/unlinked` and
`/api/prep/unmapped`; sources.js draws both lists. A fix is a line of code
(the `unmapped` skill in .claude/skills says where), so the page only
lists, and Copy hands the whole list over as text. #}
<section class="card" id="unmapped" data-section>
<div class="row-between">
<h2>Unmapped</h2>
<button type="button" id="unmapped-copy" class="secondary">Copy the list</button>
</div>
<p class="lede">What prep could not place. A place text no city or country resolves leaves its role without that place, and a host word no table maps leaves the level, contract or remote empty. Tell Claude what one is, such as "Überlingen is a town in Germany" or "Home Based - Americas is remote", or paste the copied list. The fix goes into the code, and the next pull applies it to every open role.</p>
<pre id="unmapped-text" class="copy-out" hidden></pre>
<h3>Places</h3>
<p id="unlinked-count" class="count" role="status"></p>
<div id="unlinked-list" class="srows"></div>
<h3>Host words</h3>
<p id="words-count" class="count" role="status"></p>
<div id="words-list" class="srows"></div>
</section>
{% endblock %}
{% block scripts %}
+18
View File
@@ -932,6 +932,24 @@ def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
return [UnmappedOut(**w) for w in store.unmapped()]
class UnlinkedOut(BaseModel):
"""A place text the gazetteer does not know (`Store.unlinked_places`)."""
fold: str
text: str
roles: int
countries: list[tuple[str, int]]
companies: list[str]
@router.get("/prep/unlinked")
def list_unlinked(store: StoreDep) -> list[UnlinkedOut]:
"""The place texts of open roles no place resolves, most roles first:
what a city, a spelling or a no-place word in `app/models.py` and
`app/places.py` should learn."""
return [UnlinkedOut(**p) for p in store.unlinked_places()]
class SearchBody(BaseModel):
provider: str
name: str = ""
+1
View File
@@ -86,6 +86,7 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
("job-boards", "Job boards"),
("company-boards", "Company boards"),
("runs", "Runs"),
("unmapped", "Unmapped"),
),
}