Extract as the afterburner for the level: model verdict with stated and title hints, best rank first, agreement per host

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:43:52 +02:00
1 parent 3adafd8950
commit 7ca1f02e26
4 files changed
+194 -23

No files matched your search

+23
View File
@@ -156,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser:
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
stages.add_argument("state", nargs="?", choices=["on", "off"])
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
db = sub.add_parser("db", help="database housekeeping")
@@ -182,6 +184,8 @@ def main(argv: list[str] | None = None) -> int:
return run_agents(args)
if args.command == "stages":
return run_stages(args)
if args.command == "seniority":
return run_seniority(args)
if args.command == "dupes":
return run_dupes(args)
return init_db(args)
@@ -631,6 +635,25 @@ def run_stages(args: argparse.Namespace) -> int:
return 0
def run_seniority(args: argparse.Namespace) -> int:
"""`app seniority`: per host, the roles that have both a stated and a
model level and how many of them agree, `Store.seniority_agreement`, as
a table. For deciding which host's stated level to trust."""
from app.storage import Store
config = config_mod.load(args.config)
found = Store(config.database).seniority_agreement()
if not found:
print("no role has a stated level yet")
return 0
print(f"{'host':20} {'compared':>8} {'agreed':>7} {'rate':>6} {'unread':>7}")
for h in found:
rate = "-" if h["rate"] is None else f"{h['rate']:.0%}"
print(f"{h['host'] or '(none)':20} {h['compared']:8} {h['agreed']:7} {rate:>6} "
f"{h['stated_only']:7}")
return 0
def run_dupes(args: argparse.Namespace) -> int:
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
now. Prints the groups (kept roles with a copy), the copies the feed
+29 -9
View File
@@ -7,8 +7,9 @@ Sharing the backlog out a call at a time, the give-up count, splitting a
refused batch, and failing with a notice when no model is reachable are
`desk.modeltask`'s.
A role is waiting while it is open, has never been answered (`extracted_at IS
NULL`), has not been given up on, and is in the best of an enabled agent's
`extract` is also the afterburner for the level. A role is waiting while it is
open, has not been given up on, has never been answered (`extracted_at IS
NULL`) or has no model level (`seniority_model` empty), and is in the best of an enabled agent's
ungated scores as deep and as low as that agent's own `limits` say
(`extract_top`, `extract_min`). That bound is what keeps the cost
proportional to the rankings rather than to the boards:
@@ -16,7 +17,10 @@ the pull scores every role for free first, then reads the few hundred that
rank, and scores again so the gates see what was filled. A batch built for
named uids finishes what it was dealt whatever their rank. The answer only
fills columns that are still empty: a board's structured field wins over what
a model read out of the prose. See `Store.set_extracted`.
a model read out of the prose. The exception is the level, which the model
always gives (the board's word and the title's go in as hints) and which
overrules them; both stay in `seniority_stated` and `seniority_title`, so
`Store.seniority_agreement` can compare them. See `Store.set_extracted`.
Off by default (`extract.enabled: false`), because on the API backend every
batch is a paid call. Whether it runs is `stages.stage_on`: the switch under
@@ -37,7 +41,7 @@ from desk.modeltask import ModelTask
from pipeline import NULL, RunContext, Task, job
from app.config import Config
from app.jobs import stages
from app.models import PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
from app.models import LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
from app.storage import Store
# `FUNCTIONS` is the `ats` plugin's vocabulary, reached the way `web/pages.py`
@@ -68,8 +72,16 @@ Fields:
("Berlin", "Greater London"). '' if none is named.
- country: ISO 3166-1 alpha-2 code of that place ("DE", "GB"), or ''.
- remote: "remote", "hybrid" or "onsite" only when the posting says so, else ''.
- seniority: one of {", ".join(s for s in SENIORITIES if s)}, or ''. From the
title or the stated level, not from years of experience alone.
- seniority: exactly one of {", ".join(s for s in SENIORITIES if s)}, for every
posting; never ''. This is the one field where you decide rather than
extract. Judge the level of the person hired from the title and the text (or
the short teaser when there is no text), weighing years of experience,
scope and who the role reports to. `stated` is the level the job board
itself gave and `title_level` the one a word in the title suggests. Both
are hints: boards use different vocabularies and many state nothing, so
overrule them when the posting says otherwise. When nothing says, choose the
likeliest level for that kind of title; most postings are for individual
contributors, not managers.
- salary_min, salary_max: yearly gross amounts as integers, or null. A stated
monthly figure is multiplied by 12; an hourly or daily rate is null. A single
figure is both min and max.
@@ -95,7 +107,7 @@ SCHEMA: dict[str, Any] = {
"place": _TEXT,
"country": _TEXT,
"remote": {"type": "string", "enum": list(REMOTE)},
"seniority": {"type": "string", "enum": list(SENIORITIES)},
"seniority": {"type": "string", "enum": list(LEVELS)},
"salary_min": _SALARY,
"salary_max": _SALARY,
"currency": _TEXT,
@@ -125,11 +137,15 @@ def _item(role: Role) -> dict[str, Any]:
"""One posting as the model sees it. `location` is the board's own place
text where it gave one: part of the posting, and often the only line that
names the country."""
text = " ".join(role.description.split())[:EXCERPT]
return {key: value for key, value in {
"uid": role.uid,
"title": role.title,
"location": role.place,
"text": " ".join(role.description.split())[:EXCERPT],
"stated": role.seniority_stated,
"title_level": role.seniority_title,
"text": text,
"teaser": "" if text else " ".join(role.teaser.split())[:EXCERPT],
}.items() if value}
@@ -191,7 +207,11 @@ def apply_batch(store: Store, answer: dict[str, Any],
if not uid or uid not in wanted:
stats.rejected.append(uid or "?")
continue
filled = store.set_extracted(uid, fields(one))
got = fields(one)
filled = store.set_extracted(uid, got)
if filled is not None and not got["seniority"]:
# Answered but no level: the role is still waiting for one.
store.extract_missed([uid])
if filled is None:
stats.rejected.append(uid)
continue
+47 -12
View File
@@ -51,10 +51,14 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
#: never answers is otherwise paid for on every pull for as long as it exists.
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: never answered, still open, no copy of
#: another (`rebuild_dupes`), not given up.
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL "
f"AND extract_tries < {GIVE_UP_AFTER}")
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered or without a
#: model level. The second half makes `extract` the afterburner for the level:
#: a role read before the level was asked for is read again, and only fills
#: what is still empty (`set_extracted`). A reply that names no level leaves
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
@@ -1217,7 +1221,8 @@ class Store:
otherwise only the roles in `_top_scope`."""
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")]
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1230,15 +1235,26 @@ class Store:
return int(row["n"]) if row else 0
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
"""Roles `extract` has yet to read, newest first: the best of every
enabled agent's scores, as deep and as low as that agent's own
`limits` say, so the backlog follows the rankings rather than the
boards. Given `uids`, only those of them still waiting, whatever
their rank."""
"""Roles `extract` has yet to read: the best of every enabled agent's
scores, as deep and as low as that agent's own `limits` say, so the
backlog follows the rankings rather than the boards. Ordered by the
best rank any agent gives the role (each agent's first, then each
agent's second ...), then by its score, so every agent's top comes
before anyone's tail whatever limits cut the batch. Given `uids`,
only those of them still waiting, whatever their rank; a role no agent
ranks comes last."""
where, params = self._extract_scope(uids)
return [Role.from_row(r) for r in self._rows(
f"SELECT r.* FROM roles r WHERE {where} "
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
"SELECT r.* FROM roles r LEFT JOIN ("
" SELECT uid, MIN(n) AS best_rank, MAX(score) AS best_score FROM ("
" SELECT s.uid, s.score, ROW_NUMBER() OVER ("
" PARTITION BY s.agent_id ORDER BY s.score DESC, s.uid) AS n"
" FROM scores s"
" JOIN agents a ON a.id = s.agent_id AND a.enabled = 1"
" WHERE s.gated_reason = '') GROUP BY uid) k ON k.uid = r.uid "
f"WHERE {where} "
"ORDER BY k.best_rank IS NULL, k.best_rank, k.best_score DESC, "
"r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
"""Write what `extract` read for one role and stamp `extracted_at`.
@@ -1308,6 +1324,25 @@ class Store:
f"WHERE {where}", (wanted,))
return int(crossed)
def seniority_agreement(self) -> list[dict[str, Any]]:
"""How far each host's stated level matches the model's, over the
open roles that have both: `host`, `compared`, `agreed` and `rate`
(agreed / compared), the least trusted host first. `stated_only`
counts the roles that state a level and have no model value yet, so
a small `compared` can be told from a host that states nothing."""
rows = self._rows(
f"SELECT {_HOST} AS host, "
"SUM(seniority_stated <> '' AND seniority_model <> '') AS compared, "
"SUM(seniority_stated <> '' AND seniority_stated = seniority_model) AS agreed, "
"SUM(seniority_stated <> '' AND seniority_model = '') AS stated_only "
"FROM roles WHERE closed_at IS NULL AND dup_of IS NULL "
"GROUP BY host HAVING compared > 0 OR stated_only > 0")
found = [{"host": r["host"], "compared": int(r["compared"]),
"agreed": int(r["agreed"]), "stated_only": int(r["stated_only"]),
"rate": round(r["agreed"] / r["compared"], 3) if r["compared"] else None}
for r in rows]
return sorted(found, key=lambda h: (h["rate"] is None, h["rate"] or 0, h["host"]))
# ------------------------------------------------------------ summarise
#: The `state` key counting replies that left a role without a summary.
+95 -2
View File
@@ -136,8 +136,8 @@ def test_closed_and_answered_roles_are_not_offered(
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'done'")
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"seniority_model = 'senior' WHERE source_id = 'done'")
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
modeltask.run(extract.TASK, config, store)
@@ -215,3 +215,96 @@ def test_disabled_finds_nothing_and_asks_nobody(
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
assert model.asked == [] and result["extracted"] == 0
def test_a_role_read_before_without_a_level_is_read_again_for_it(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1", seniority="senior")
store.upsert_role(item)
ranked(store, item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"country = 'DE' WHERE source_id = '1'")
stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff")
for r in roles])
modeltask.run(extract.TASK, config, store)
role = store.role(item.uid)
assert role is not None
assert (role.seniority_model, role.seniority) == ("staff", "staff")
assert role.seniority_stated == "senior", "the host's word stays beside the model's"
assert role.country == "DE", "the other fields still only fill what is empty"
assert store.extract_pending() == 0
def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
withtext = posting("text", title="Senior Engineer", seniority="mid")
noText = posting("teaser", title="Analyst")
noText.body = ""
store.upsert_role(withtext)
store.upsert_role(noText)
with store.transaction() as conn:
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
"WHERE source_id = 'teaser'")
ranked(store, withtext, noText)
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles])
modeltask.run(extract.TASK, config, store)
asked = {r["uid"]: r for batch in model.asked for r in batch}
one, two = asked[withtext.uid], asked[noText.uid]
assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one
assert two["teaser"] == "Join our analytics team" and "text" not in two
assert "stated" not in two and "overrule" in extract.SYSTEM
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
assert "" not in schema["enum"], "the afterburner must pick a level"
def test_an_answer_with_no_level_is_charged_until_given_up(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
item = posting("1")
store.upsert_role(item)
ranked(store, item)
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 1, "answered without a level: still waiting"
modeltask.run(extract.TASK, config, store)
modeltask.run(extract.TASK, config, store)
assert store.extract_pending() == 0
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
for item in items:
store.upsert_role(item)
one = store.save_agent("One", "python backend")
two = store.save_agent("Two", "python backend")
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
(items[2].uid, 0.7, "")])
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
got = [r.uid for r in store.extract_waiting(10)]
assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid]
def test_agreement_per_host(store: Store) -> None:
for sid, board, stated, model in [
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
("5", "lever", "senior", "")]:
store.upsert_role(posting(sid, board=board, seniority=stated))
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
(model, sid))
got = {h["host"]: h for h in store.seniority_agreement()}
assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \
== (2, 1, 0.5)
assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \
== (1, 1.0, 1)
assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]