List unlinked places and unmapped words under Sources, fix them in code with the unmapped skill
Settings > Sources > Unmapped shows the place texts no place resolves, with their roles' countries and companies, and the host words no table maps, and copies both as text. Prep fingerprints places.seed(), so a town or spelling added to the code is seeded and every open role is mapped again at the next prep. app prep unlinked lists the places on the command line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ae978594b4
commit
06ff9f549b
13 files changed
+313
-43
No files matched your search
+16
-5
@@ -8,7 +8,7 @@
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app dupes
|
||||
app prep [unmapped] [--remap]
|
||||
app prep [unmapped|unlinked] [--remap]
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -162,9 +162,10 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
prep = sub.add_parser(
|
||||
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
|
||||
prep.add_argument("action", nargs="?", choices=["unmapped"],
|
||||
help="list the host words no table maps, instead of running")
|
||||
"prep", help="map the roles' host words and fold the copies, or list what is unmapped")
|
||||
prep.add_argument("action", nargs="?", choices=["unmapped", "unlinked"],
|
||||
help="list the host words no table maps, or the place texts no "
|
||||
"place resolves, instead of running")
|
||||
prep.add_argument("--remap", action="store_true",
|
||||
help="map every role again, closed ones included")
|
||||
|
||||
@@ -679,7 +680,9 @@ def run_dupes(args: argparse.Namespace) -> int:
|
||||
def run_prep(args: argparse.Namespace) -> int:
|
||||
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
||||
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
||||
host words no table maps, per field and host, most roles first."""
|
||||
host words no table maps, per field and host, most roles first; `app
|
||||
prep unlinked` the place texts of open roles no place resolves, with the
|
||||
countries and companies of their roles."""
|
||||
from app import prep
|
||||
from app.storage import Store
|
||||
|
||||
@@ -692,6 +695,14 @@ def run_prep(args: argparse.Namespace) -> int:
|
||||
for w in found:
|
||||
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
||||
return 0
|
||||
if args.action == "unlinked":
|
||||
linked = store.unlinked_places()
|
||||
if not linked:
|
||||
print("every place text is linked")
|
||||
for p in linked:
|
||||
where = " ".join(f"{c or '-'}:{n}" for c, n in p["countries"][:3])
|
||||
print(f"{p['roles']:6} {p['text'][:40]:40} {where:20} {', '.join(p['companies'])}")
|
||||
return 0
|
||||
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
||||
return 0
|
||||
|
||||
|
||||
+13
-8
@@ -4,9 +4,11 @@ A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||
and marks the role due. `run` maps every due role: its company through
|
||||
`company_aliases`, its places against the gazetteer, its level, contract
|
||||
and remote through the host's tables (`app.prep.words`), and its country
|
||||
from the places (`app.prep.country`). When the tables changed since the
|
||||
last run (`fingerprint`, kept in state as `prep:tables`), every open role
|
||||
is mapped again from its stored words, with no new pull. Then the words no
|
||||
from the places (`app.prep.country`). When the tables or the gazetteer in
|
||||
code (`places.seed`) changed since the last run (`fingerprint`, kept in
|
||||
state as `prep:tables`), the new places and spellings are added to the
|
||||
database and every open role is mapped again from its stored words, with
|
||||
no new pull. Then the words no
|
||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||
copies across the whole database.
|
||||
"""
|
||||
@@ -49,9 +51,10 @@ class Prepped(NamedTuple):
|
||||
|
||||
|
||||
def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
|
||||
sort_keys=True)
|
||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||
code describes and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||
"places": repr(places.seed())}, sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
|
||||
@@ -114,12 +117,14 @@ def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
||||
|
||||
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||
dupes: bool = True) -> Prepped:
|
||||
"""Map the due roles, or every open one when the tables changed, or
|
||||
every role with `remap`; list the unmapped words; fold the copies."""
|
||||
"""Map the due roles, or every open one when the tables or the
|
||||
gazetteer changed, or every role with `remap`; list the unmapped words;
|
||||
fold the copies."""
|
||||
mark = fingerprint(tables)
|
||||
whole = remap or store.state.get(STATE) != mark
|
||||
rows = store.prep_rows("due")
|
||||
if whole:
|
||||
store.seed_places()
|
||||
due = {r["uid"] for r in rows}
|
||||
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
||||
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||
|
||||
@@ -1331,6 +1331,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
gap: .4rem; font-size: .8rem; color: var(--dim);
|
||||
}
|
||||
.srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; }
|
||||
/* A count beside a row's name, as the Unmapped rows show their roles. */
|
||||
.srow > .count { margin: 0; white-space: nowrap; }
|
||||
/* A company row's Set board / Change editor: its own line under `.sub`,
|
||||
wrapping like `.sub` does, the boxes styled as `.source-form`'s. */
|
||||
.crow .board-edit {
|
||||
@@ -1360,6 +1362,16 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
.prune > summary { cursor: pointer; color: var(--dim); font-weight: 600; }
|
||||
.prune[open] > summary { margin-bottom: .75rem; }
|
||||
|
||||
/* Where the Unmapped list goes when the clipboard is refused, as news's
|
||||
`.share-url`: outside a secure context (the server at a LAN address) there
|
||||
is none, and text has to be on the page to be selectable. */
|
||||
.copy-out {
|
||||
margin: .5rem 0 1rem; padding: .5rem .7rem; max-height: 16rem; overflow: auto;
|
||||
font-family: var(--mono); font-size: .78rem; white-space: pre-wrap;
|
||||
overflow-wrap: anywhere; background: var(--bg); border: 1px solid var(--line);
|
||||
border-radius: .3rem;
|
||||
}
|
||||
|
||||
/* applications and companies */
|
||||
|
||||
table.list { width: 100%; border-collapse: collapse; background: var(--panel); }
|
||||
|
||||
+109
-29
@@ -19,9 +19,14 @@
|
||||
* Under the Follow form of Company boards, a fold lists what each reader
|
||||
* declares (`/api/readers`): how its list arrives, where the text is, what
|
||||
* it filters at the source, how it states places, and the robots.txt
|
||||
* decision taken when it was built. A second fold lists the host words no
|
||||
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
|
||||
* host, most roles first; it is left out when there are none.
|
||||
* decision taken when it was built.
|
||||
*
|
||||
* The Unmapped section lists what prep could not place, most roles first:
|
||||
* the place texts no place resolves (`/api/prep/unlinked`), with the
|
||||
* countries and companies of their roles, and the host words no table in
|
||||
* `app/prep/hosts` maps (`/api/prep/unmapped`), per field and host. Copy
|
||||
* puts both on the clipboard as text for Claude, or, where there is no
|
||||
* clipboard, on the page selected.
|
||||
*/
|
||||
(function () {
|
||||
"use strict";
|
||||
@@ -322,36 +327,108 @@
|
||||
$("#board-form").after(fold);
|
||||
}
|
||||
|
||||
/** The host words prep could not map, one row per field and host. */
|
||||
function renderUnmapped(words) {
|
||||
if (!words.length) return;
|
||||
// ------------------------------------------------------------ unmapped
|
||||
|
||||
/** Rows shown before Show all; the copy always holds every one. */
|
||||
const UNLINKED_SHOWN = 40;
|
||||
/** @type {{places: object[], words: object[]}} */
|
||||
const unmapped = { places: [], words: [] };
|
||||
|
||||
/** "DE 5 · EU 5 · none 1", the three biggest. */
|
||||
function where(place) {
|
||||
const shown = place.countries.slice(0, 3).map(([code, n]) => `${code || "none"} ${n}`);
|
||||
if (place.countries.length > 3) shown.push("…");
|
||||
return shown.join(" · ");
|
||||
}
|
||||
|
||||
function srow(name, right, sub) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "srow";
|
||||
const label = document.createElement("span");
|
||||
label.className = "name";
|
||||
label.textContent = name;
|
||||
const n = document.createElement("span");
|
||||
n.className = "count";
|
||||
n.textContent = right;
|
||||
const line = document.createElement("span");
|
||||
line.className = "sub";
|
||||
line.textContent = sub;
|
||||
row.append(label, n, line);
|
||||
return row;
|
||||
}
|
||||
|
||||
function renderUnlinked(all) {
|
||||
const list = $("#unlinked-list");
|
||||
const shown = all ? unmapped.places : unmapped.places.slice(0, UNLINKED_SHOWN);
|
||||
list.replaceChildren(...shown.map((p) => srow(p.text, count(p.roles, "role"),
|
||||
[where(p), p.companies.join(", ")].filter(Boolean).join(" · "))));
|
||||
$("#unlinked-count").textContent = unmapped.places.length
|
||||
? `${count(unmapped.places.length, "place text")} on open roles no place resolves.`
|
||||
: "Every place text of an open role resolves.";
|
||||
if (shown.length < unmapped.places.length) {
|
||||
const more = document.createElement("button");
|
||||
more.type = "button";
|
||||
more.className = "linky";
|
||||
more.textContent = `Show all ${unmapped.places.length}`;
|
||||
more.addEventListener("click", () => renderUnlinked(true));
|
||||
list.appendChild(more);
|
||||
}
|
||||
}
|
||||
|
||||
function renderWords() {
|
||||
const groups = new Map();
|
||||
for (const w of words) {
|
||||
for (const w of unmapped.words) {
|
||||
const key = `${w.field} · ${w.host || "no host"}`;
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(w);
|
||||
}
|
||||
const fold = document.createElement("details");
|
||||
fold.className = "prune";
|
||||
fold.id = "unmapped";
|
||||
const summary = document.createElement("summary");
|
||||
summary.textContent = `Host words no table maps (${words.length})`;
|
||||
const list = document.createElement("div");
|
||||
list.className = "srows";
|
||||
for (const [key, found] of groups) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "srow";
|
||||
const name = document.createElement("span");
|
||||
name.className = "name";
|
||||
name.textContent = key;
|
||||
const sub = document.createElement("span");
|
||||
sub.className = "sub";
|
||||
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
|
||||
row.append(name, sub);
|
||||
list.appendChild(row);
|
||||
$("#words-list").replaceChildren(...[...groups].map(([key, found]) =>
|
||||
srow(key, count(found.reduce((n, w) => n + w.roles, 0), "role"),
|
||||
found.map((w) => `${w.word} ${w.roles}`).join(" · "))));
|
||||
$("#words-count").textContent = unmapped.words.length
|
||||
? `${count(unmapped.words.length, "host word")} no table maps.`
|
||||
: "Every host word is mapped.";
|
||||
}
|
||||
|
||||
/** Both lists as text, one line an entry, for pasting to Claude. */
|
||||
function unmappedText() {
|
||||
const lines = ["Unlinked places (roles, their countries, companies):"];
|
||||
for (const p of unmapped.places) {
|
||||
lines.push(`${p.roles} ${p.text} [${where(p)}] ${p.companies.join(", ")}`);
|
||||
}
|
||||
fold.append(summary, list);
|
||||
($("#readers") || $("#board-form")).after(fold);
|
||||
lines.push("", "Unmapped host words (field, host, roles, word):");
|
||||
for (const w of unmapped.words) {
|
||||
lines.push(`${w.field} ${w.host || "-"} ${w.roles} ${w.word}`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
// The clipboard needs a secure context; the server reached at a LAN
|
||||
// address is not one. As news's copyLink, the text is then put on the
|
||||
// page and selected, for the device's own Copy.
|
||||
async function copyUnmapped() {
|
||||
const text = unmappedText();
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
window.toast.good("Copied the unmapped list.");
|
||||
return;
|
||||
} catch { /* no clipboard here: show the text instead */ }
|
||||
const holder = $("#unmapped-text");
|
||||
holder.textContent = text;
|
||||
holder.hidden = false;
|
||||
const range = document.createRange();
|
||||
range.selectNodeContents(holder);
|
||||
const selection = window.getSelection();
|
||||
selection.removeAllRanges();
|
||||
selection.addRange(range);
|
||||
window.toast.info("The list is selected: use your device's Copy.");
|
||||
}
|
||||
|
||||
async function loadUnmapped() {
|
||||
[unmapped.places, unmapped.words] = await Promise.all(
|
||||
[api("/api/prep/unlinked"), api("/api/prep/unmapped")]);
|
||||
renderUnlinked(false);
|
||||
renderWords();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------ one section shown
|
||||
@@ -395,8 +472,11 @@
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
if (!$("#searches-list")) return;
|
||||
if ($("#board-form")) {
|
||||
api("/api/readers").then(renderReaders, () => {})
|
||||
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
|
||||
api("/api/readers").then(renderReaders, () => {});
|
||||
}
|
||||
if ($("#unlinked-list")) {
|
||||
loadUnmapped().catch(() => {});
|
||||
$("#unmapped-copy").addEventListener("click", copyUnmapped);
|
||||
}
|
||||
try {
|
||||
await load();
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -601,6 +602,35 @@ class Store:
|
||||
gazetteer.relink(conn, gaz, folded)
|
||||
return place_id
|
||||
|
||||
def unlinked_places(self, limit: int | None = None) -> list[dict[str, Any]]:
|
||||
"""`unresolved_places` with what places each text: `countries`, the
|
||||
`[code, roles]` prep gave its roles ('' for none), and `companies`,
|
||||
the three that write it most, both most roles first. What the Sources
|
||||
page's Unmapped section and `app prep unlinked` list."""
|
||||
found = self.unresolved_places(limit=limit)
|
||||
countries: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
|
||||
companies: dict[str, Counter[str]] = {fold: Counter() for fold, _, _ in found}
|
||||
for r in self._rows(
|
||||
"""SELECT rp.fold, r.country, r.company, COUNT(DISTINCT r.uid) AS n
|
||||
FROM role_places rp JOIN roles r ON r.uid = rp.uid
|
||||
WHERE rp.place_id IS NULL AND r.closed_at IS NULL AND r.dup_of IS NULL
|
||||
GROUP BY rp.fold, r.country, r.company"""):
|
||||
if (fold := str(r["fold"])) in countries:
|
||||
countries[fold][str(r["country"])] += int(r["n"])
|
||||
companies[fold][str(r["company"])] += int(r["n"])
|
||||
return [{"fold": fold, "text": text, "roles": n,
|
||||
"countries": [list(c) for c in countries[fold].most_common()],
|
||||
"companies": [c for c, _ in companies[fold].most_common(3) if c]}
|
||||
for fold, text, n in found]
|
||||
|
||||
def seed_places(self) -> int:
|
||||
"""Add what `places.seed` describes and the database lacks, a place
|
||||
or a spelling written into the code since the file was made. Prep
|
||||
calls it when the code changed and resolves the open roles again.
|
||||
Returns how many places were added."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.seed(conn)
|
||||
|
||||
def relink_places(self) -> int:
|
||||
"""Resolve every role's own places again, after the gazetteer was
|
||||
seeded anew. Returns how many roles now resolve differently."""
|
||||
|
||||
@@ -163,6 +163,25 @@
|
||||
<div id="notices"><span class="muted">loading…</span></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{# What prep could not place, from `/api/prep/unlinked` and
|
||||
`/api/prep/unmapped`; sources.js draws both lists. A fix is a line of code
|
||||
(the `unmapped` skill in .claude/skills says where), so the page only
|
||||
lists, and Copy hands the whole list over as text. #}
|
||||
<section class="card" id="unmapped" data-section>
|
||||
<div class="row-between">
|
||||
<h2>Unmapped</h2>
|
||||
<button type="button" id="unmapped-copy" class="secondary">Copy the list</button>
|
||||
</div>
|
||||
<p class="lede">What prep could not place. A place text no city or country resolves leaves its role without that place, and a host word no table maps leaves the level, contract or remote empty. Tell Claude what one is, such as "Überlingen is a town in Germany" or "Home Based - Americas is remote", or paste the copied list. The fix goes into the code, and the next pull applies it to every open role.</p>
|
||||
<pre id="unmapped-text" class="copy-out" hidden></pre>
|
||||
<h3>Places</h3>
|
||||
<p id="unlinked-count" class="count" role="status"></p>
|
||||
<div id="unlinked-list" class="srows"></div>
|
||||
<h3>Host words</h3>
|
||||
<p id="words-count" class="count" role="status"></p>
|
||||
<div id="words-list" class="srows"></div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
{% block scripts %}
|
||||
|
||||
@@ -932,6 +932,24 @@ def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
|
||||
return [UnmappedOut(**w) for w in store.unmapped()]
|
||||
|
||||
|
||||
class UnlinkedOut(BaseModel):
|
||||
"""A place text the gazetteer does not know (`Store.unlinked_places`)."""
|
||||
|
||||
fold: str
|
||||
text: str
|
||||
roles: int
|
||||
countries: list[tuple[str, int]]
|
||||
companies: list[str]
|
||||
|
||||
|
||||
@router.get("/prep/unlinked")
|
||||
def list_unlinked(store: StoreDep) -> list[UnlinkedOut]:
|
||||
"""The place texts of open roles no place resolves, most roles first:
|
||||
what a city, a spelling or a no-place word in `app/models.py` and
|
||||
`app/places.py` should learn."""
|
||||
return [UnlinkedOut(**p) for p in store.unlinked_places()]
|
||||
|
||||
|
||||
class SearchBody(BaseModel):
|
||||
provider: str
|
||||
name: str = ""
|
||||
|
||||
@@ -86,6 +86,7 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
|
||||
("job-boards", "Job boards"),
|
||||
("company-boards", "Company boards"),
|
||||
("runs", "Runs"),
|
||||
("unmapped", "Unmapped"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user