diff --git a/app/cli.py b/app/cli.py index de75961..7ea0f1c 100644 --- a/app/cli.py +++ b/app/cli.py @@ -156,6 +156,8 @@ def build_parser() -> argparse.ArgumentParser: stages.add_argument("name", nargs="?", choices=["extract", "summarise"]) stages.add_argument("state", nargs="?", choices=["on", "off"]) + sub.add_parser("seniority", help="per host, how often the stated level equals the model's") + sub.add_parser("dupes", help="recompute which open roles are copies of one posting") db = sub.add_parser("db", help="database housekeeping") @@ -182,6 +184,8 @@ def main(argv: list[str] | None = None) -> int: return run_agents(args) if args.command == "stages": return run_stages(args) + if args.command == "seniority": + return run_seniority(args) if args.command == "dupes": return run_dupes(args) return init_db(args) @@ -631,6 +635,25 @@ def run_stages(args: argparse.Namespace) -> int: return 0 +def run_seniority(args: argparse.Namespace) -> int: + """`app seniority`: per host, the roles that have both a stated and a + model level and how many of them agree, `Store.seniority_agreement`, as + a table. For deciding which host's stated level to trust.""" + from app.storage import Store + + config = config_mod.load(args.config) + found = Store(config.database).seniority_agreement() + if not found: + print("no role has a stated level yet") + return 0 + print(f"{'host':20} {'compared':>8} {'agreed':>7} {'rate':>6} {'unread':>7}") + for h in found: + rate = "-" if h["rate"] is None else f"{h['rate']:.0%}" + print(f"{h['host'] or '(none)':20} {h['compared']:8} {h['agreed']:7} {rate:>6} " + f"{h['stated_only']:7}") + return 0 + + def run_dupes(args: argparse.Namespace) -> int: """`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`, now. Prints the groups (kept roles with a copy), the copies the feed diff --git a/app/jobs/extract.py b/app/jobs/extract.py index d1d9b65..456f760 100644 --- a/app/jobs/extract.py +++ b/app/jobs/extract.py @@ -7,8 +7,9 @@ Sharing the backlog out a call at a time, the give-up count, splitting a refused batch, and failing with a notice when no model is reachable are `desk.modeltask`'s. -A role is waiting while it is open, has never been answered (`extracted_at IS -NULL`), has not been given up on, and is in the best of an enabled agent's +`extract` is also the afterburner for the level. A role is waiting while it is +open, has not been given up on, has never been answered (`extracted_at IS +NULL`) or has no model level (`seniority_model` empty), and is in the best of an enabled agent's ungated scores as deep and as low as that agent's own `limits` say (`extract_top`, `extract_min`). That bound is what keeps the cost proportional to the rankings rather than to the boards: @@ -16,7 +17,10 @@ the pull scores every role for free first, then reads the few hundred that rank, and scores again so the gates see what was filled. A batch built for named uids finishes what it was dealt whatever their rank. The answer only fills columns that are still empty: a board's structured field wins over what -a model read out of the prose. See `Store.set_extracted`. +a model read out of the prose. The exception is the level, which the model +always gives (the board's word and the title's go in as hints) and which +overrules them; both stay in `seniority_stated` and `seniority_title`, so +`Store.seniority_agreement` can compare them. See `Store.set_extracted`. Off by default (`extract.enabled: false`), because on the API backend every batch is a paid call. Whether it runs is `stages.stage_on`: the switch under @@ -37,7 +41,7 @@ from desk.modeltask import ModelTask from pipeline import NULL, RunContext, Task, job from app.config import Config from app.jobs import stages -from app.models import PERMITS, REMOTE, SENIORITIES, ExtractResult, Role +from app.models import LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role from app.storage import Store # `FUNCTIONS` is the `ats` plugin's vocabulary, reached the way `web/pages.py` @@ -68,8 +72,16 @@ Fields: ("Berlin", "Greater London"). '' if none is named. - country: ISO 3166-1 alpha-2 code of that place ("DE", "GB"), or ''. - remote: "remote", "hybrid" or "onsite" only when the posting says so, else ''. -- seniority: one of {", ".join(s for s in SENIORITIES if s)}, or ''. From the - title or the stated level, not from years of experience alone. +- seniority: exactly one of {", ".join(s for s in SENIORITIES if s)}, for every + posting; never ''. This is the one field where you decide rather than + extract. Judge the level of the person hired from the title and the text (or + the short teaser when there is no text), weighing years of experience, + scope and who the role reports to. `stated` is the level the job board + itself gave and `title_level` the one a word in the title suggests. Both + are hints: boards use different vocabularies and many state nothing, so + overrule them when the posting says otherwise. When nothing says, choose the + likeliest level for that kind of title; most postings are for individual + contributors, not managers. - salary_min, salary_max: yearly gross amounts as integers, or null. A stated monthly figure is multiplied by 12; an hourly or daily rate is null. A single figure is both min and max. @@ -95,7 +107,7 @@ SCHEMA: dict[str, Any] = { "place": _TEXT, "country": _TEXT, "remote": {"type": "string", "enum": list(REMOTE)}, - "seniority": {"type": "string", "enum": list(SENIORITIES)}, + "seniority": {"type": "string", "enum": list(LEVELS)}, "salary_min": _SALARY, "salary_max": _SALARY, "currency": _TEXT, @@ -125,11 +137,15 @@ def _item(role: Role) -> dict[str, Any]: """One posting as the model sees it. `location` is the board's own place text where it gave one: part of the posting, and often the only line that names the country.""" + text = " ".join(role.description.split())[:EXCERPT] return {key: value for key, value in { "uid": role.uid, "title": role.title, "location": role.place, - "text": " ".join(role.description.split())[:EXCERPT], + "stated": role.seniority_stated, + "title_level": role.seniority_title, + "text": text, + "teaser": "" if text else " ".join(role.teaser.split())[:EXCERPT], }.items() if value} @@ -191,7 +207,11 @@ def apply_batch(store: Store, answer: dict[str, Any], if not uid or uid not in wanted: stats.rejected.append(uid or "?") continue - filled = store.set_extracted(uid, fields(one)) + got = fields(one) + filled = store.set_extracted(uid, got) + if filled is not None and not got["seniority"]: + # Answered but no level: the role is still waiting for one. + store.extract_missed([uid]) if filled is None: stats.rejected.append(uid) continue diff --git a/app/storage/store.py b/app/storage/store.py index 4f8d184..79e6ab5 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -51,10 +51,14 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") #: never answers is otherwise paid for on every pull for as long as it exists. GIVE_UP_AFTER = 3 -#: A role `extract` has yet to read: never answered, still open, no copy of -#: another (`rebuild_dupes`), not given up. -_EXTRACT_WAITING = (f"extracted_at IS NULL AND closed_at IS NULL AND dup_of IS NULL " - f"AND extract_tries < {GIVE_UP_AFTER}") +#: A role `extract` has yet to read: still open, no copy of another +#: (`rebuild_dupes`), not given up, and either never answered or without a +#: model level. The second half makes `extract` the afterburner for the level: +#: a role read before the level was asked for is read again, and only fills +#: what is still empty (`set_extracted`). A reply that names no level leaves +#: the role waiting and is charged a try (`extract.apply_batch`), so it ends. +_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL " + f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}") #: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged. VOTES = (-1, 0, 1, None) @@ -1217,7 +1221,8 @@ class Store: otherwise only the roles in `_top_scope`.""" where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at") .replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries") - .replace("dup_of", "r.dup_of")] + .replace("dup_of", "r.dup_of") + .replace("seniority_model", "r.seniority_model")] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] @@ -1230,15 +1235,26 @@ class Store: return int(row["n"]) if row else 0 def extract_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]: - """Roles `extract` has yet to read, newest first: the best of every - enabled agent's scores, as deep and as low as that agent's own - `limits` say, so the backlog follows the rankings rather than the - boards. Given `uids`, only those of them still waiting, whatever - their rank.""" + """Roles `extract` has yet to read: the best of every enabled agent's + scores, as deep and as low as that agent's own `limits` say, so the + backlog follows the rankings rather than the boards. Ordered by the + best rank any agent gives the role (each agent's first, then each + agent's second ...), then by its score, so every agent's top comes + before anyone's tail whatever limits cut the batch. Given `uids`, + only those of them still waiting, whatever their rank; a role no agent + ranks comes last.""" where, params = self._extract_scope(uids) return [Role.from_row(r) for r in self._rows( - f"SELECT r.* FROM roles r WHERE {where} " - "ORDER BY r.first_seen DESC, r.uid LIMIT ?", [*params, limit])] + "SELECT r.* FROM roles r LEFT JOIN (" + " SELECT uid, MIN(n) AS best_rank, MAX(score) AS best_score FROM (" + " SELECT s.uid, s.score, ROW_NUMBER() OVER (" + " PARTITION BY s.agent_id ORDER BY s.score DESC, s.uid) AS n" + " FROM scores s" + " JOIN agents a ON a.id = s.agent_id AND a.enabled = 1" + " WHERE s.gated_reason = '') GROUP BY uid) k ON k.uid = r.uid " + f"WHERE {where} " + "ORDER BY k.best_rank IS NULL, k.best_rank, k.best_score DESC, " + "r.first_seen DESC, r.uid LIMIT ?", [*params, limit])] def set_extracted(self, uid: str, fields: dict[str, Any]) -> int | None: """Write what `extract` read for one role and stamp `extracted_at`. @@ -1308,6 +1324,25 @@ class Store: f"WHERE {where}", (wanted,)) return int(crossed) + def seniority_agreement(self) -> list[dict[str, Any]]: + """How far each host's stated level matches the model's, over the + open roles that have both: `host`, `compared`, `agreed` and `rate` + (agreed / compared), the least trusted host first. `stated_only` + counts the roles that state a level and have no model value yet, so + a small `compared` can be told from a host that states nothing.""" + rows = self._rows( + f"SELECT {_HOST} AS host, " + "SUM(seniority_stated <> '' AND seniority_model <> '') AS compared, " + "SUM(seniority_stated <> '' AND seniority_stated = seniority_model) AS agreed, " + "SUM(seniority_stated <> '' AND seniority_model = '') AS stated_only " + "FROM roles WHERE closed_at IS NULL AND dup_of IS NULL " + "GROUP BY host HAVING compared > 0 OR stated_only > 0") + found = [{"host": r["host"], "compared": int(r["compared"]), + "agreed": int(r["agreed"]), "stated_only": int(r["stated_only"]), + "rate": round(r["agreed"] / r["compared"], 3) if r["compared"] else None} + for r in rows] + return sorted(found, key=lambda h: (h["rate"] is None, h["rate"] or 0, h["host"])) + # ------------------------------------------------------------ summarise #: The `state` key counting replies that left a role without a summary. diff --git a/tests/test_extract.py b/tests/test_extract.py index 5549a25..df8c28c 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -136,8 +136,8 @@ def test_closed_and_answered_roles_are_not_offered( with store.transaction() as conn: conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' " "WHERE source_id = 'closed'") - conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00' " - "WHERE source_id = 'done'") + conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " + "seniority_model = 'senior' WHERE source_id = 'done'") model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles]) modeltask.run(extract.TASK, config, store) @@ -215,3 +215,96 @@ def test_disabled_finds_nothing_and_asks_nobody( result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store) assert model.asked == [] and result["extracted"] == 0 + + +def test_a_role_read_before_without_a_level_is_read_again_for_it( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + item = posting("1", seniority="senior") + store.upsert_role(item) + ranked(store, item) + with store.transaction() as conn: + conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " + "country = 'DE' WHERE source_id = '1'") + stub(monkeypatch, lambda roles: [answer(r["uid"], country="FR", seniority="staff") + for r in roles]) + + modeltask.run(extract.TASK, config, store) + + role = store.role(item.uid) + assert role is not None + assert (role.seniority_model, role.seniority) == ("staff", "staff") + assert role.seniority_stated == "senior", "the host's word stays beside the model's" + assert role.country == "DE", "the other fields still only fill what is empty" + assert store.extract_pending() == 0 + + +def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + withtext = posting("text", title="Senior Engineer", seniority="mid") + noText = posting("teaser", title="Analyst") + noText.body = "" + store.upsert_role(withtext) + store.upsert_role(noText) + with store.transaction() as conn: + conn.execute("UPDATE roles SET teaser = 'Join our analytics team' " + "WHERE source_id = 'teaser'") + ranked(store, withtext, noText) + model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid") for r in roles]) + + modeltask.run(extract.TASK, config, store) + + asked = {r["uid"]: r for batch in model.asked for r in batch} + one, two = asked[withtext.uid], asked[noText.uid] + assert (one["stated"], one["title_level"]) == ("mid", "senior") and "teaser" not in one + assert two["teaser"] == "Join our analytics team" and "text" not in two + assert "stated" not in two and "overrule" in extract.SYSTEM + schema = extract.SCHEMA["properties"]["roles"]["items"]["properties"]["seniority"] + assert "" not in schema["enum"], "the afterburner must pick a level" + + +def test_an_answer_with_no_level_is_charged_until_given_up( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + item = posting("1") + store.upsert_role(item) + ranked(store, item) + stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles]) + + modeltask.run(extract.TASK, config, store) + assert store.extract_pending() == 1, "answered without a level: still waiting" + modeltask.run(extract.TASK, config, store) + modeltask.run(extract.TASK, config, store) + assert store.extract_pending() == 0 + + +def test_each_agents_best_come_first_then_the_rest(store: Store) -> None: + items = [posting(s) for s in ("a1", "a2", "a3", "b1")] + for item in items: + store.upsert_role(item) + one = store.save_agent("One", "python backend") + two = store.save_agent("Two", "python backend") + store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""), + (items[2].uid, 0.7, "")]) + store.replace_scores(two.id, [(items[3].uid, 0.6, "")]) + + got = [r.uid for r in store.extract_waiting(10)] + + assert got == [items[0].uid, items[3].uid, items[1].uid, items[2].uid] + + +def test_agreement_per_host(store: Store) -> None: + for sid, board, stated, model in [ + ("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"), + ("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"), + ("5", "lever", "senior", "")]: + store.upsert_role(posting(sid, board=board, seniority=stated)) + with store.transaction() as conn: + conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?", + (model, sid)) + + got = {h["host"]: h for h in store.seniority_agreement()} + + assert (got["personio"]["compared"], got["personio"]["agreed"], got["personio"]["rate"]) \ + == (2, 1, 0.5) + assert (got["lever"]["compared"], got["lever"]["rate"], got["lever"]["stated_only"]) \ + == (1, 1.0, 1) + assert [h["host"] for h in store.seniority_agreement()] == ["personio", "lever"]