Merge branch 'wp-i-seniority'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
7896adad27
4 files changed
+194
-23
No files matched your search
+23
@@ -156,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
|
||||
stages.add_argument("state", nargs="?", choices=["on", "off"])
|
||||
|
||||
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
|
||||
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
@@ -182,6 +184,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_agents(args)
|
||||
if args.command == "stages":
|
||||
return run_stages(args)
|
||||
if args.command == "seniority":
|
||||
return run_seniority(args)
|
||||
if args.command == "dupes":
|
||||
return run_dupes(args)
|
||||
return init_db(args)
|
||||
@@ -631,6 +635,25 @@ def run_stages(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_seniority(args: argparse.Namespace) -> int:
|
||||
"""`app seniority`: per host, the roles that have both a stated and a
|
||||
model level and how many of them agree, `Store.seniority_agreement`, as
|
||||
a table. For deciding which host's stated level to trust."""
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
found = Store(config.database).seniority_agreement()
|
||||
if not found:
|
||||
print("no role has a stated level yet")
|
||||
return 0
|
||||
print(f"{'host':20} {'compared':>8} {'agreed':>7} {'rate':>6} {'unread':>7}")
|
||||
for h in found:
|
||||
rate = "-" if h["rate"] is None else f"{h['rate']:.0%}"
|
||||
print(f"{h['host'] or '(none)':20} {h['compared']:8} {h['agreed']:7} {rate:>6} "
|
||||
f"{h['stated_only']:7}")
|
||||
return 0
|
||||
|
||||
|
||||
def run_dupes(args: argparse.Namespace) -> int:
|
||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||
|
||||
+29
-9
@@ -7,8 +7,9 @@ Sharing the backlog out a call at a time, the give-up count, splitting a
|
||||
refused batch, and failing with a notice when no model is reachable are
|
||||
`desk.modeltask`'s.
|
||||
|
||||
A role is waiting while it is open, has never been answered (`extracted_at IS
|
||||
NULL`), has not been given up on, and is in the best of an enabled agent's
|
||||
`extract` is also the afterburner for the level. A role is waiting while it is
|
||||
open, has not been given up on, has never been answered (`extracted_at IS
|
||||
NULL`) or has no model level (`seniority_model` empty), and is in the best of an enabled agent's
|
||||
ungated scores as deep and as low as that agent's own `limits` say
|
||||
(`extract_top`, `extract_min`). That bound is what keeps the cost
|
||||
proportional to the rankings rather than to the boards:
|
||||
@@ -16,7 +17,10 @@ the pull scores every role for free first, then reads the few hundred that
|
||||
rank, and scores again so the gates see what was filled. A batch built for
|
||||
named uids finishes what it was dealt whatever their rank. The answer only
|
||||
fills columns that are still empty: a board's structured field wins over what
|
||||
a model read out of the prose. See `Store.set_extracted`.
|
||||
a model read out of the prose. The exception is the level, which the model
|
||||
always gives (the board's word and the title's go in as hints) and which
|
||||
overrules them; both stay in `seniority_stated` and `seniority_title`, so
|
||||
`Store.seniority_agreement` can compare them. See `Store.set_extracted`.
|
||||
|
||||
Off by default (`extract.enabled: false`), because on the API backend every
|
||||
batch is a paid call. Whether it runs is `stages.stage_on`: the switch under
|
||||
@@ -37,7 +41,7 @@ from desk.modeltask import ModelTask
|
||||
from pipeline import NULL, RunContext, Task, job
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.models import PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
|
||||
from app.models import LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
|
||||
from app.storage import Store
|
||||
|
||||
# `FUNCTIONS` is the `ats` plugin's vocabulary, reached the way `web/pages.py`
|
||||
@@ -68,8 +72,16 @@ Fields:
|
||||
("Berlin", "Greater London"). '' if none is named.
|
||||
- country: ISO 3166-1 alpha-2 code of that place ("DE", "GB"), or ''.
|
||||
- remote: "remote", "hybrid" or "onsite" only when the posting says so, else ''.
|
||||
- seniority: one of {", ".join(s for s in SENIORITIES if s)}, or ''. From the
|
||||
title or the stated level, not from years of experience alone.
|
||||
- seniority: exactly one of {", ".join(s for s in SENIORITIES if s)}, for every
|
||||
posting; never ''. This is the one field where you decide rather than
|
||||
extract. Judge the level of the person hired from the title and the text (or
|
||||
the short teaser when there is no text), weighing years of experience,
|
||||
scope and who the role reports to. `stated` is the level the job board
|
||||
itself gave and `title_level` the one a word in the title suggests. Both
|
||||
are hints: boards use different vocabularies and many state nothing, so
|
||||
overrule them when the posting says otherwise. When nothing says, choose the
|
||||
likeliest level for that kind of title; most postings are for individual
|
||||
contributors, not managers.
|
||||
- salary_min, salary_max: yearly gross amounts as integers, or null. A stated
|
||||
monthly figure is multiplied by 12; an hourly or daily rate is null. A single
|
||||
figure is both min and max.
|
||||
@@ -95,7 +107,7 @@ SCHEMA: dict[str, Any] = {
|
||||
"place": _TEXT,
|
||||
"country": _TEXT,
|
||||
"remote": {"type": "string", "enum": list(REMOTE)},
|
||||
"seniority": {"type": "string", "enum": list(SENIORITIES)},
|
||||
"seniority": {"type": "string", "enum": list(LEVELS)},
|
||||
"salary_min": _SALARY,
|
||||
"salary_max": _SALARY,
|
||||
"currency": _TEXT,
|
||||
@@ -125,11 +137,15 @@ def _item(role: Role) -> dict[str, Any]:
|
||||
"""One posting as the model sees it. `location` is the board's own place
|
||||
text where it gave one: part of the posting, and often the only line that
|
||||
names the country."""
|
||||
text = " ".join(role.description.split())[:EXCERPT]
|
||||
return {key: value for key, value in {
|
||||
"uid": role.uid,
|
||||
"title": role.title,
|
||||
"location": role.place,
|
||||
"text": " ".join(role.description.split())[:EXCERPT],
|
||||
"stated": role.seniority_stated,
|
||||
"title_level": role.seniority_title,
|
||||
"text": text,
|
||||
"teaser": "" if text else " ".join(role.teaser.split())[:EXCERPT],
|
||||
}.items() if value}
|
||||
|
||||
|
||||
@@ -191,7 +207,11 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
||||
if not uid or uid not in wanted:
|
||||
stats.rejected.append(uid or "?")
|
||||
continue
|
||||
filled = store.set_extracted(uid, fields(one))
|
||||
got = fields(one)
|
||||
filled = store.set_extracted(uid, got)
|
||||
if filled is not None and not got["seniority"]:
|
||||
# Answered but no level: the role is still waiting for one.
|
||||
store.extract_missed([uid])
|
||||
if filled is None:
|
||||
stats.rejected.append(uid)
|
||||
continue
|
||||
|
||||
+47
-12
@@ -51,10 +51,14 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
#: never answers is otherwise paid for on every pull for as long as it exists.
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: never answered, still open, no copy of
|
||||
#: another (`rebuild_dupes`), not given up.
|
||||
_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL "
|
||||
f"AND extract_tries < {GIVE_UP_AFTER}")
|
||||
#: A role `extract` has yet to read: still open, no copy of another
|
||||
#: (`rebuild_dupes`), not given up, and either never answered or without a
|
||||
#: model level. The second half makes `extract` the afterburner for the level:
|
||||
#: a role read before the level was asked for is read again, and only fills
|
||||
#: what is still empty (`set_extracted`). A reply that names no level leaves
|
||||
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
|
||||
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
|
||||
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
VOTES = (-1, 0, 1, None)
|
||||
@@ -1217,7 +1221,8 @@ class Store:
|
||||
otherwise only the roles in `_top_scope`."""
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")]
|
||||
.replace("dup_of", "r.dup_of")
|
||||
.replace("seniority_model", "r.seniority_model")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
@@ -1230,15 +1235,26 @@ class Store:
|
||||
return int(row["n"]) if row else 0
|
||||
|
||||
def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||
"""Roles `extract` has yet to read, newest first: the best of every
|
||||
enabled agent's scores, as deep and as low as that agent's own
|
||||
`limits` say, so the backlog follows the rankings rather than the
|
||||
boards. Given `uids`, only those of them still waiting, whatever
|
||||
their rank."""
|
||||
"""Roles `extract` has yet to read: the best of every enabled agent's
|
||||
scores, as deep and as low as that agent's own `limits` say, so the
|
||||
backlog follows the rankings rather than the boards. Ordered by the
|
||||
best rank any agent gives the role (each agent's first, then each
|
||||
agent's second ...), then by its score, so every agent's top comes
|
||||
before anyone's tail whatever limits cut the batch. Given `uids`,
|
||||
only those of them still waiting, whatever their rank; a role no agent
|
||||
ranks comes last."""
|
||||
where, params = self._extract_scope(uids)
|
||||
return [Role.from_row(r) for r in self._rows(
|
||||
f"SELECT r.* FROM roles r WHERE {where} "
|
||||
"ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
||||
"SELECT r.* FROM roles r LEFT JOIN ("
|
||||
" SELECT uid, MIN(n) AS best_rank, MAX(score) AS best_score FROM ("
|
||||
" SELECT s.uid, s.score, ROW_NUMBER() OVER ("
|
||||
" PARTITION BY s.agent_id ORDER BY s.score DESC, s.uid) AS n"
|
||||
" FROM scores s"
|
||||
" JOIN agents a ON a.id = s.agent_id AND a.enabled = 1"
|
||||
" WHERE s.gated_reason = '') GROUP BY uid) k ON k.uid = r.uid "
|
||||
f"WHERE {where} "
|
||||
"ORDER BY k.best_rank IS NULL, k.best_rank, k.best_score DESC, "
|
||||
"r.first_seen DESC, r.uid LIMIT ?", [*params, limit])]
|
||||
|
||||
def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None:
|
||||
"""Write what `extract` read for one role and stamp `extracted_at`.
|
||||
@@ -1308,6 +1324,25 @@ class Store:
|
||||
f"WHERE {where}", (wanted,))
|
||||
return int(crossed)
|
||||
|
||||
def seniority_agreement(self) -> list[dict[str, Any]]:
|
||||
"""How far each host's stated level matches the model's, over the
|
||||
open roles that have both: `host`, `compared`, `agreed` and `rate`
|
||||
(agreed / compared), the least trusted host first. `stated_only`
|
||||
counts the roles that state a level and have no model value yet, so
|
||||
a small `compared` can be told from a host that states nothing."""
|
||||
rows = self._rows(
|
||||
f"SELECT {_HOST} AS host, "
|
||||
"SUM(seniority_stated <> '' AND seniority_model <> '') AS compared, "
|
||||
"SUM(seniority_stated <> '' AND seniority_stated = seniority_model) AS agreed, "
|
||||
"SUM(seniority_stated <> '' AND seniority_model = '') AS stated_only "
|
||||
"FROM roles WHERE closed_at IS NULL AND dup_of IS NULL "
|
||||
"GROUP BY host HAVING compared > 0 OR stated_only > 0")
|
||||
found = [{"host": r["host"], "compared": int(r["compared"]),
|
||||
"agreed": int(r["agreed"]), "stated_only": int(r["stated_only"]),
|
||||
"rate": round(r["agreed"] / r["compared"], 3) if r["compared"] else None}
|
||||
for r in rows]
|
||||
return sorted(found, key=lambda h: (h["rate"] is None, h["rate"] or 0, h["host"]))
|
||||
|
||||
# ------------------------------------------------------------ summarise
|
||||
|
||||
#: The `state` key counting replies that left a role without a summary.
|
||||
|
||||
+95
-2
@@ -136,8 +136,8 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||
"WHERE source_id = 'closed'")
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00' "
|
||||
"WHERE source_id = 'done'")
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
"seniority_model = 'senior' WHERE source_id = 'done'")
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
@@ -215,3 +215,96 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
||||
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
||||
|
||||
assert model.asked == [] and result["extracted"] == 0
|
||||
|
||||
|
||||
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", seniority="senior")
|
||||
store.upsert_role(item)
|
||||
ranked(store, item)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
"country = 'DE' WHERE source_id = '1'")
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff")
|
||||
for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
role = store.role(item.uid)
|
||||
assert role is not None
|
||||
assert (role.seniority_model, role.seniority) == ("staff", "staff")
|
||||
assert role.seniority_stated == "senior", "the host's word stays beside the model's"
|
||||
assert role.country == "DE", "the other fields still only fill what is empty"
|
||||
assert store.extract_pending() == 0
|
||||
|
||||
|
||||
def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
||||
noText = posting("teaser", title="Analyst")
|
||||
noText.body = ""
|
||||
store.upsert_role(withtext)
|
||||
store.upsert_role(noText)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
||||
"WHERE source_id = 'teaser'")
|
||||
ranked(store, withtext, noText)
|
||||
model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
asked = {r["uid"]: r for batch in model.asked for r in batch}
|
||||
one, two = asked[withtext.uid], asked[noText.uid]
|
||||
assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one
|
||||
assert two["teaser"] == "Join our analytics team" and "text" not in two
|
||||
assert "stated" not in two and "overrule" in extract.SYSTEM
|
||||
schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"]
|
||||
assert "" not in schema["enum"], "the afterburner must pick a level"
|
||||
|
||||
|
||||
def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1")
|
||||
store.upsert_role(item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
||||
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending() == 1, "answered without a level: still waiting"
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
assert store.extract_pending() == 0
|
||||
|
||||
|
||||
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
||||
for item in items:
|
||||
store.upsert_role(item)
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend")
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||
(items[2].uid, 0.7, "")])
|
||||
store.replace_scores(two.id, [(items[3].uid, 0.6, "")])
|
||||
|
||||
got = [r.uid for r in store.extract_waiting(10)]
|
||||
|
||||
assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid]
|
||||
|
||||
|
||||
def test_agreement_per_host(store: Store) -> None:
|
||||
for sid, board, stated, model in [
|
||||
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
||||
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
||||
("5", "lever", "senior", "")]:
|
||||
store.upsert_role(posting(sid, board=board, seniority=stated))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
||||
(model, sid))
|
||||
|
||||
got = {h["host"]: h for h in store.seniority_agreement()}
|
||||
|
||||
assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \
|
||||
== (2, 1, 0.5)
|
||||
assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \
|
||||
== (1, 1.0, 1)
|
||||
assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]
|
||||
Reference in new issue
Block a user