Merge main into wp-review-losses

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:54:33 +02:00
commit fac3487925
27 files changed
+774 -325

No files matched your search

+1 -1
View File
@@ -6,7 +6,7 @@
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
app searches add|list|remove
app agents list|prompt
app stages [extract|summarise on|off]
app stages [extract|summarise|enrich on|off]
app dupes
app prep [unmapped|unlinked] [--remap]
app db init
+21 -3
View File
@@ -16,8 +16,9 @@ the starter agents; `config.example.yaml` documents every key.
from __future__ import annotations
import os
import sys
from pathlib import Path
from typing import Any, Literal
from typing import Any
import yaml
from pydantic import Field, ValidationError, field_validator, model_validator
@@ -165,12 +166,29 @@ class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna."""
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
`scope="search"`, so a new one needs no edit here."""
name: str
provider: Literal["jobindex", "adzuna"]
provider: str
params: dict[str, Any] = Field(default_factory=dict)
@field_validator("provider")
@classmethod
def _a_search_provider(cls, provider: str) -> str:
# `plugins` is a top-level folder beside `app` and is not on the path
# when this module is imported first, so it is reached the way
# `app.models` reaches it (`_declared_posted`), and only when a seed
# is validated.
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import PROVIDERS
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
return provider
class Config(Section):
"""The whole file."""
+13 -7
View File
@@ -9,6 +9,8 @@ good for a given search. For each agent:
column filled for at least one, is gated ("none of
country DK / remote remote")
prefer gates `weight` when the role matches, 0 when it does not
exclude gates a role that matches one of the values keeps its row, with
score 0 and the value ("title account executive")
A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it. `extract` is what fills an empty column, and it reads
@@ -29,7 +31,8 @@ roles out of extract's reach for good.
Every term is 0..1. The score is the weighted sum over the sum of the weights
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
company (`affinity = 2`) skips the `any` group but not the `require` gates, and
company (`affinity = 2`) skips the `any` group but not the `require` or
`exclude` gates, and
is scored like any other company with its affinity term at 1: nothing puts
its roles above a better fit. A hidden one (`-1`) is not scored at all. A
role with no vector is scored without `fit` and `taste`, unless `fit` is all
@@ -156,14 +159,17 @@ def _reason(gate: Gate) -> str:
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
"""Why this role is gated, as reasons: every require gate it fails, and
the `any` group when it matches none of them. A gate with an empty column
(state -1) neither passes nor fails, so the group fails only when some
member had evidence. A favourite company's roles skip the `any` group but
not the require gates, so a star widens a search to a company's roles
within e.g. Europe, not to the whole world."""
"""Why this role is gated, as reasons: every require gate it fails, every
exclude gate it matches, and the `any` group when it matches none of them.
A gate with an empty column (state -1) neither passes nor fails, so the
group fails only when some member had evidence. A favourite company's
roles skip the `any` group but not the require and exclude gates, so a
star widens a search to a company's roles within e.g. Europe, not to the
whole world, and not to the titles the agent leaves out."""
states = list(zip(gates, candidate.gates, strict=True))
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
if g.mode == "exclude" and state > 0]
group = [(g, state) for g, state in states if g.mode == "any"]
if candidate.affinity != 2 and group and not any(state == 1 for _, state in group) \
and any(state == 0 for _, state in group):
+6 -3
View File
@@ -11,11 +11,14 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`stage:enrich`, set the same way as the others: by the Runs page
(`PUT /api/stages/enrich`) and by `app stages enrich on|off`.
`describe` is what `GET /api/stages` and `app stages` show: for each stage
with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
would answer.
would answer. The model is the one a call is made with: `agent.model` when
set, since it overrides the per-task model everywhere (`desk/modeltask.py`),
else the stage's own.
"""
from __future__ import annotations
@@ -98,7 +101,7 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(getattr(config, name), "model", ""), backend=backend,
model=config.agent.model or getattr(config, name).model, backend=backend,
answer=said)
+12 -5
View File
@@ -1167,21 +1167,26 @@ class Application(Model):
#: The role columns a gate can name. `company` matches on the company key;
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title`
#: matches a value as whole words anywhere in the title, folded as `title_key`
#: folds it, so `account executive` finds "Enterprise Account Executive, UK"
#: and `data engineer` does not find "Data Engineering Manager"; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"]
"currency", "permit", "languages", "company", "title"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer"]
GateMode = Literal["require", "any", "prefer", "exclude"]
class Gate(Model):
"""One condition of an agent. `require` gates a role that fails it (the
row is kept with the reason); the `any` gates together are one group a
role must match at least one of ("in Denmark, or remote"); `prefer` adds
`weight` to its score when it matches. A gate rejects on evidence only: a
`weight` to its score when it matches; `exclude` gates a role that matches
any of its values ("title account executive"), a favourite's too. A gate
rejects on evidence only: a
role whose column is empty is not rejected, since `extract` fills the
column later for the roles that rank, and only for those."""
@@ -1297,7 +1302,9 @@ class Agent(Model):
class Candidate(Model):
"""What scoring reads about one open role for one agent: its posted
date (for `recency`), its company's affinity for that agent, and per
gate `1` match, `0` no match, `-1` the column is empty."""
gate `1` match, `0` no match, `-1` the column is empty. An `exclude`
gate's match is the position of the first value it matched, from 1, so
the reason can name it."""
uid: str
posted_at: str
+13 -4
View File
@@ -39,8 +39,8 @@
"HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
"IS", "LI", "NO", "GB", "CH", "EU"];
const FIELDS = ["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"];
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"]];
"currency", "permit", "languages", "company", "title"];
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"], ["exclude", "not"]];
const HYBRID_WEIGHT = 0.05;
/** @type {object[]} every agent, as the server lists them */
@@ -227,11 +227,14 @@
const must = of("require").map(phrase);
const prefer = of("prefer").map((g) => phrase(g).replace(/^with /, ""));
if (where.hybrid) prefer.unshift("hybrid");
const not = of("exclude").map(phrase);
const sentences = [];
const any = whereSentence(where, of("any"));
if (any) sentences.push(any);
if (must.length) sentences.push(`Only roles ${must.join(" and ")}.`);
if (prefer.length) sentences.push(`Prefers ${prefer.join("; ")}.`);
if (where.titles) sentences.push(`Leaves out titles with ${orList(where.titles.values.map((t) => `“${t}”`))}.`);
if (not.length) sentences.push(`Leaves out roles ${not.join(" or ")}.`);
return sentences.join(" ");
}
@@ -497,7 +500,7 @@
// that is Europe and the countries line exactly, and only with Remote
// ticked; any other must on country is a row, and Remote reads anywhere.
function distribute(gates) {
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null };
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null, titles: null };
const rest = [];
const musts = [];
const plain = (g) => g.weight === undefined || g.weight === 1;
@@ -512,6 +515,7 @@
else if (g.field === "remote" && g.mode === "any" && plain(g) && only(g, "remote")) slot = "remote";
else if (g.field === "place" && g.mode === "any" && plain(g)) slot = "places";
else if (g.field === "remote" && g.mode === "prefer" && only(g, "hybrid")) slot = "hybrid";
else if (g.field === "title" && g.mode === "exclude" && plain(g)) slot = "titles";
if (slot && !where[slot]) where[slot] = g;
else rest.push(g);
}
@@ -561,7 +565,9 @@
const { where, rest } = distribute(row ? row.gates : []);
const countries = countryInput(where.countries ? where.countries.values : []);
const places = wordInput(where.places ? where.places.values : [], "Add a town");
const titles = wordInput(where.titles ? where.titles.values : [], "account executive");
const terms = wordInput(row ? row.fetch_terms || [] : [], "machine learning, data science");
$("#a-titles", f).replaceWith(titles.node);
$("#a-terms", f).replaceWith(terms.node);
$("#a-countries", f).replaceWith(countries.node);
$("#a-towns", f).replaceWith(places.node);
@@ -601,7 +607,8 @@
});
// The gates the Where block says, in the order the starter agents have
// them: the must on country, the lines, then hybrid.
// them: the must on country, the lines, then hybrid; and the titles left
// out, the block under it.
function whereGates() {
const out = [];
const within = countries.values();
@@ -615,6 +622,8 @@
if ($("#a-hybrid", f).checked) {
out.push({ field: "remote", values: ["hybrid"], mode: "prefer", weight: Number($("#a-hybrid-weight", f).value) });
}
const left = titles.values();
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
return out;
}
// The gates Save sends: the Where block's, then the rows. The sentence
+2 -1
View File
@@ -1223,7 +1223,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
border-radius: .3rem; padding: .35rem .5rem;
}
/* The fieldsets that hold rows: a column, the Add button under the rows. */
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.terms,
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
.agent-form fieldset.terms,
.agent-form fieldset.rules {
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
border: 1px solid var(--line); border-radius: .3rem;
+3 -2
View File
@@ -59,7 +59,7 @@
const company = companies.get(application.company_key);
cell(tr, "Company", company ? company.name : application.company_key);
const select = statusSelect(application.status);
select.addEventListener("change", () => setStatus(application.id, select));
select.addEventListener("change", () => setStatus(application.id, select, application.status));
cell(tr, "Status", select);
const saved = cell(tr, "Saved", savedOn(application.created_at));
saved.title = application.created_at;
@@ -69,7 +69,7 @@
count(applications.length, "tracked application");
}
async function setStatus(id, select) {
async function setStatus(id, select, was) {
select.disabled = true;
try {
await api(`/api/applications/${encodeURIComponent(id)}/status`, {
@@ -77,6 +77,7 @@
});
render(await api("/api/applications"));
} catch (err) {
select.value = was; // the change was refused, so the row still holds `was`
select.disabled = false;
}
}
+4 -4
View File
@@ -217,8 +217,8 @@
// Replacing a different board closes its roles (see set_board in
// app/jobs/sources.py); ask first, naming both boards and the count,
// the same as Detach and a removed search already do.
if (company.board && company.board !== board
&& !window.confirm(`Replace ${company.name}'s board (${company.board}) `
if (company.target && company.target !== board
&& !window.confirm(`Replace ${company.name}'s board (${company.target}) `
+ `with ${board}? Its ${count(company.open_roles, "open role")} close.`)) {
return;
}
@@ -429,8 +429,8 @@
// editor does, so it asks first too.
const existing = [...companies.values()]
.find((c) => c.name.toLowerCase() === name.toLowerCase());
if (existing && existing.board && existing.board !== board
&& !window.confirm(`Replace ${existing.name}'s board (${existing.board}) `
if (existing && existing.target && existing.target !== board
&& !window.confirm(`Replace ${existing.name}'s board (${existing.target}) `
+ `with ${board}? Its ${count(existing.open_roles, "open role")} close.`)) {
return;
}
+11 -4
View File
@@ -109,10 +109,17 @@
...(stage.model ? [el("span", "badge", stage.model)] : []));
row.append(name, stageSwitch(stage, () => renderStages(stages)));
const sub = el("span", "sub");
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
const agents = el("a", "", "Search agents");
agents.href = "/settings/agents";
sub.append(agents, stage.source === "config"
if (stage.name === "enrich") {
/* Global, unlike the model stages: `enrich.top` and `enrich.min_score`. */
sub.append(`${DOES[stage.name]}. How many it reads is set for every agent alike by `
+ "enrich.top and enrich.min_score in config.yaml");
} else {
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
const agents = el("a", "", "Search agents");
agents.href = "/settings/agents";
sub.append(agents);
}
sub.append(stage.source === "config"
? ". Following config.yaml until this switch is used."
: ".");
row.appendChild(sub);
+3 -2
View File
@@ -864,7 +864,8 @@
is the only time this page ever sees a role's `summary` or
`description` directly. */
function excerptOf(role) {
const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim();
/* The server's order (`_EXCERPT_SOURCE`): summary, description, teaser. */
const text = (role.summary || role.description || role.teaser || "").replace(/\s+/g, " ").trim();
if (text.length <= 200) return text;
const cut = text.slice(0, 200);
const space = cut.lastIndexOf(" ");
@@ -1458,7 +1459,7 @@
async function fetchText(button) {
const uid = state.readingUid;
const role = await busy(button, "Fetching…",
() => api(`/api/roles/${encodeURIComponent(uid)}/text`, { method: "POST" }));
() => api(roleUrl(uid).replace(/(\?|$)/, "/text$1"), { method: "POST" }));
if (role) merge(role);
}
+1 -1
View File
@@ -176,7 +176,7 @@
text: n.text,
fix: n.fix,
detail: n.detail,
link: { label: "all messages", href: "/messages" },
link: { label: "all messages", href: "/settings/sources#messages" },
};
if (PROBLEM.has(n.level)) t.count = n.count;
if (n.level === "good") t.action = { label: "reload", run: () => location.reload() };
+16 -4
View File
@@ -219,6 +219,13 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
if gate.field == "place":
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "title":
# Whole words of the folded title: " account executive " inside
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
# value cannot smuggle a LIKE wildcard in.
words = [f"% {title_key(v)} %" for v in values if title_key(v)]
match = " OR ".join("(' ' || r.title_key || ' ') LIKE ?" for _ in words) or "0"
return "r.title_key = ''", f"({match})", words
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -1830,10 +1837,15 @@ class Store:
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
# An exclude gate says which of its values matched: one WHEN each.
parts = ([gate.model_copy(update={"values": [v]}) for v in gate.values]
if gate.mode == "exclude" else [gate])
whens = []
for k, part in enumerate(parts, 1):
empty, match, values = _gate_sql(part, gaz)
whens.append(f"WHEN {match} THEN {k}")
params += values
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
rows = self._rows(
+8
View File
@@ -49,6 +49,14 @@
<span class="hint">This one ranks rather than filters: a hybrid role gains a little.</span>
<p id="a-where-sum" class="where-sum" role="status"></p>
</fieldset>
<fieldset class="titles">
<legend>Which titles to leave out?</legend>
<div class="where-line">
<span>Leave out titles with</span>
<div id="a-titles"></div>
</div>
<span class="hint">Separate phrases with a comma or Enter. A phrase matches whole words anywhere in the title, whatever the case: <code>account executive</code> takes "Enterprise Account Executive, UK", and <code>data engineer</code> leaves "Data Engineering Manager" alone. A shorter phrase takes more: <code>sales</code> takes Sales Engineer too. It applies to liked companies as well. A role left out keeps its row, and Show gated on Roles lists it with the phrase that took it.</span>
</fieldset>
<fieldset class="terms">
<legend>What big boards send</legend>
<div class="where-line">
+30 -8
View File
@@ -98,7 +98,13 @@ def _feed_filters(
raise HTTPException(422, f"{name} is one of {', '.join(RUNGS)}, not {rung!r}.")
if track and track not in TRACKS:
raise HTTPException(422, f"track is one of {', '.join(TRACKS)}, not {track!r}.")
levels = levels_between(level_from, level_to, track) if level_from or level_to or track else []
levels: tuple[str, ...] = ()
if level_from or level_to or track:
# The store reads an empty list as "no filter", so a range that holds no
# level must be refused here or it would list every role.
levels = levels_between(level_from, level_to, track)
if not levels:
raise HTTPException(422, "That seniority range holds no level.")
return {
"q": q or None, "company": company or None, "place": place or None,
"function": function or None, "remote": remote or None, "days": days,
@@ -359,14 +365,21 @@ def _known_role(store: Store, uid: str) -> Role:
@router.post("/roles/{uid}/text")
def fetch_text(uid: str, store: StoreDep) -> Role:
def fetch_text(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
"""Fetch the posting page once and keep it as plain text, for a role whose
description is a teaser (`jobs.posting.fetch_posting`); a Jobindex role
is read from the page its "Se jobbet" link leads to."""
is read from the page its "Se jobbet" link leads to. The role comes back as
`agent` sees it, as `GET /api/roles/{uid}` does: the page merges the answer
into the open role, and one without the vote would blank the reader's."""
agent_id = _agent_id(store, agent)
try:
return jobs.posting.fetch_posting(store, uid)
jobs.posting.fetch_posting(store, uid)
except jobs.posting.PostingError as exc:
raise HTTPException(exc.status, str(exc)) from exc
role = store.role(uid, agent_id)
if role is None:
raise HTTPException(404, f"no such role: {uid}")
return role
@router.post("/roles/{uid}/summarise", status_code=202)
@@ -436,12 +449,17 @@ class CompanyOut(BaseModel):
with no board shown that is not `from_search`, and `hand_reason` says why
it is read by hand (`app.models.HAND_REASONS`). `all_countries` and
`all_roles` are the company's two fetch overrides: its big boards read
with the agents' wide reach, and with no role-type narrowing."""
with the agents' wide reach, and with no role-type narrowing. `target` is
the attached board in full, `host:slug`, where `board` is its host alone;
the page compares it with the board typed to know whether a replacement
closes anything. `ats_slug` cannot stand in for it, since the careers site
may run on another host than the board."""
key: str
name: str
notes: str = ""
board: str
target: str = ""
subscription_id: int | None = None
on: bool | None = None
affinity: int
@@ -500,6 +518,7 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
sub = subs.get(sub_id) if sub_id is not None else None
from_search = jobs.sources.known_from_search_only(c, ever_boarded)
out.append(CompanyOut(key=c.key, name=c.name, notes=c.notes, board=host,
target=target,
subscription_id=sub_id, on=on, affinity=c.affinity,
open_roles=counts.get(c.key, 0), ats=c.ats,
ats_slug=c.ats_slug, careers_url=c.careers_url,
@@ -865,9 +884,12 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
sub = store.subscription(sub_id)
if sub is None or sub.source != "ats":
raise HTTPException(404, f"no such board: {sub_id}")
codes = sorted({c.strip().upper() for c in body.countries if c.strip()})
if unknown := [c for c in codes if not places.known(f"country:{c}") or c == "EU"]:
raise HTTPException(422, f"Not a country code: {', '.join(unknown)}.")
named = sorted({c.strip() for c in body.countries if c.strip()})
tokens = {c: places.parse(f"country:{c}") for c in named}
if unknown := [c for c, t in tokens.items() if not places.known(str(t)) or t.value == "EU"]:
raise HTTPException(422, f"Not a country: {', '.join(unknown)}.")
# The ISO code is what the reach compares against, not the name or alias typed.
codes = sorted({t.value for t in tokens.values()})
if body.mode == "own" and not (codes or body.remote):
raise HTTPException(422, "Name a country or remote, or take the agents' reach.")
params = {k: v for k, v in sub.params.items() if k not in jobs.reach.OVERRIDE_KEYS}