diff --git a/app/jobs/judge.py b/app/jobs/judge.py index 98a5fee..9b63419 100644 --- a/app/jobs/judge.py +++ b/app/jobs/judge.py @@ -14,8 +14,8 @@ career count as at most `app.models.BROKEN_CAP` (4), when the role is served. Culture and location are left out: the first is guessed from ad text, the second is what the fetch gate does. -The work is pairs, `(agent, role)`: an enabled agent with a CV and a prompt, -and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by +The work is pairs, `(agent, role)`: an enabled agent with a CV, with or +without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by fit, the best by CV closeness that those leave out, and the liked and saved) with some text, that has no judgement yet, or one read from another CV, prompts, like, avoid or must text or role text than today's (`input_key`). @@ -189,10 +189,10 @@ def input_key(agent: Agent, text: str) -> str: def _judged(agent: Agent) -> bool: - """Whether `judge` reads for this agent at all: enabled, with a CV and a - prompt with text (without one nothing of its cut is in by fit).""" - return agent.enabled and bool(agent.cv_text.strip()) \ - and any(p.text.strip() for p in agent.prompts) + """Whether `judge` reads for this agent at all: enabled, with a CV. An + agent without a prompt is judged too: its cut is the CV list and the + kept, and the profile goes without `` (`system`).""" + return agent.enabled and bool(agent.cv_text.strip()) def waiting(store: Store, keys: Sequence[str] | None = None, diff --git a/app/jobs/score.py b/app/jobs/score.py index 90e1249..9e19e14 100644 --- a/app/jobs/score.py +++ b/app/jobs/score.py @@ -9,16 +9,19 @@ good for a given search. Each agent first applies its fetch gate: lifts any gates one group, the place part: a role that matches none of them, and has the column filled for at least one, is - gated ("none of country DK / remote remote"); skipped for - a company set to All locations + gated ("none of country DK / remote remote"), unless its + location is unknown; skipped for a company set to All + locations exclude gates a role that matches one of the values keeps its row, with score 0 and the value ("title account executive"); those on `title` are skipped for a company set to All roles A gate rejects on evidence, never on absence: a role whose column is empty -is not gated by it. `extract` is what fills an empty column, and it reads -only the roles in the cut, so gating on an empty column would keep those -roles out of extract's reach for good. A hidden company (`affinity = -1`) +is not gated by it, and a place the gazetteer could not resolve, or one +that holds the agent's places (Øresund for Copenhagen), is no evidence. +`extract` is what fills an empty column, and it reads only the roles in +the cut, so gating on an empty column would keep those roles out of +extract's reach for good. A hidden company (`affinity = -1`) is not scored at all; a favourite passes the gate like any other company. The final score, the feed's order, is the weighted sum of four terms over @@ -181,6 +184,10 @@ def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]: it fails, every exclude gate it matches, and the `any` group when it matches none of them. A gate with an empty column (state -1) neither passes nor fails, so the group fails only when some member had evidence. + A role whose location is unknown, every `country` and `place` member + empty, passes the group whatever its remote says: a hybrid role with no + place may still be in one of the agent's towns, and extract reads it + only in the cut. A group of remote alone still fails a hybrid role. A company set to All locations skips the `any` group, the place part, and one set to All roles the exclude gates on `title`, the titles to leave out; the require gates, the region, apply to every company.""" @@ -190,7 +197,10 @@ def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]: if g.mode == "exclude" and state > 0 and not (candidate.all_roles and g.field == "title")] group = [(g, state) for g, state in states if g.mode == "any"] - if not candidate.all_countries and group and not any(state == 1 for _, state in group) \ + located = [state for g, state in group if g.field in ("country", "place")] + unknown = bool(located) and all(state == -1 for state in located) + if not candidate.all_countries and group and not unknown \ + and not any(state == 1 for _, state in group) \ and any(state == 0 for _, state in group): reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group)) return reasons diff --git a/app/storage/store.py b/app/storage/store.py index 000931a..75e1e9f 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -65,7 +65,8 @@ _EXTRACT_TEXT = ("place", "country", "currency", "permit") GIVE_UP_AFTER = 3 #: A role `extract` has yet to read: still open, no copy of another -#: (`rebuild_dupes`), not given up, and either never answered, without a +#: (`rebuild_dupes`), not given up, and either never answered (or answered +#: before a better text came, `upsert_role`), without a #: model level, never asked where a remote hire may live #: (`remote_where_model` NULL), or never asked for its languages and deadline #: (`languages_required` NULL). The last three make `extract` the @@ -83,6 +84,15 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe _IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id " "AND a.enabled = 1 WHERE s.cut <> '')") +#: A role one agent's feed shows, as a `WHERE` on `r` taking the agent's id +#: twice: its fetch gate let the role through (`_roles_from` with +#: `fetch_gate`), and the company is not one it hid. With the open and kept +#: copy test, what the rail's counts for that agent cover. +_SHOWN = ("EXISTS (SELECT 1 FROM scores s WHERE s.uid = r.uid AND s.agent_id = ? " + "AND s.gated_reason = '') " + "AND r.company_key NOT IN (SELECT company_key FROM agent_companies " + "WHERE agent_id = ? AND affinity = -1)") + #: The order within one agent's cut, over its scores as `s`: in by fit, best #: fit first, then in by CV closeness, closest first, then kept. _CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, " @@ -289,14 +299,22 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: `params` belong to `match`; `empty` takes none. `country` and `place` match over the gazetteer, as the feed's Place - filter does (`_places_sql`): a country gate is met by a role whose + filter does (`_places_sql`), and also on a role place that holds one of + the gate's places (`wider`): a country gate is met by a role whose `country` is one of its codes or that has a place in one of them, so - `DK` is met by `Copenhagen; London`. Its column is empty only when the - role has neither. A `require` gate on country reads a remote role whose + `DK` is met by `Copenhagen; London`, and by Øresund or Europe alone, + which are no evidence the role is outside Denmark; a town in Germany + is, also beside Europe. Its column is empty only when the role has neither a country + nor a resolved place. A `require` gate on country reads a remote role whose area is stated through the area alone, as the remote chips do (`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)" meet Europe, "Remote US" does not, and neither is empty. A place gate's - value is a place token, bare text a `place:`. + value is a place token, bare text a `place:`. Its column is empty when + the role has no place the gazetteer resolved, as "Storkøbenhavn" or + "Headquarters" are not: that is no evidence the role is elsewhere. Such + a role can still match, by its country or by whole words of its line + for a town the gazetteer does not know, so a place gate's `match` is + tested before its `empty` (`Store.score_candidates`). `remote_from` takes place tokens without their `remote:` (`country:DK`, `anywhere`) and matches as the remote chips do. It is empty for a role @@ -314,7 +332,8 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params if gate.field == "country": tokens = [f"country:{v}" for v in values] - match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values]) + match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values], + wider=True) empty = f"(r.country = '' AND {_unplaced(gaz)})" if gate.mode != "require": return empty, match, params @@ -323,8 +342,8 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: return (f"({empty} AND NOT {_HAS_AREA})", f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params]) if gate.field == "place": - match, params = _places_sql(values, gaz, table="r") - return "r.place = ''", match, params + match, params = _places_sql(values, gaz, table="r", wider=True) + return _unplaced(gaz), match, params if gate.field == "title": # Whole words of the folded title: " account executive " inside # " enterprise account executive uk ". A fold drops `%` and `_`, so a @@ -344,15 +363,23 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles", - codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]: + codes: Sequence[str] = (), scope: Sequence[Gate] = (), + wider: bool = False) -> tuple[str, list[Any]]: """Any of the place tokens over `table`, as a join over the gazetteer: a role with a place under one of the places the tokens name (`country:DK` every place in Denmark, `region:skane` every town in Skåne, `place:malmo` Malmö by any spelling), or whose `country` is one of their countries (a role whose board gave a country and no place). `codes` are more values of - `country` to accept. A `place:` token the gazetteer does not know is - matched on whole words of the role's folded line (`places.match_sql`), - so a town nobody has entered yet is still found. + `country` to accept. `wider` also takes a role placed only at places + that hold one of them (`Gazetteer.overlap`): Øresund, Europe or Anywhere + alone meet `country:DK`, as the fetch gate reads them (`_gate_sql`); the + feed's Place filter leaves it off. A role that also names a place + elsewhere is not taken: `London; Europe` is in London, and `Aarhus; + Denmark` does not meet Copenhagen. The `country` column is not widened, + as Sweden would then meet Malmö for a role in Stockholm. A `place:` + token the gazetteer does not know is matched on whole words of the + role's folded line (`places.match_sql`), so a town nobody has entered + yet is still found. A `remote:` token is a remote role whose area overlaps its places (`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere. @@ -360,6 +387,7 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r `scope`, the viewing agent's `require` gates on country; with none, every remote role.""" ids: set[int] = set() + around: set[int] = set() areas: set[int] = set() found = {c for c in codes if c} clauses: list[str] = [] @@ -382,15 +410,26 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r continue under = gaz.under(named) ids |= under + if wider: + around |= gaz.overlap(named) found |= gaz.codes(under) + first: list[tuple[str, list[Any]]] = [] if ids: - clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places " - "WHERE place_id IN (SELECT value FROM json_each(?)))") - params.insert(0, json.dumps(sorted(ids))) + first.append((f"{table}.uid IN (SELECT uid FROM role_places " + "WHERE place_id IN (SELECT value FROM json_each(?)))", + [json.dumps(sorted(ids))])) if found: - clauses.insert(1 if ids else 0, - f"{table}.country IN (SELECT value FROM json_each(?))") - params.insert(1 if ids else 0, json.dumps(sorted(found))) + first.append((f"{table}.country IN (SELECT value FROM json_each(?))", + [json.dumps(sorted(found))])) + if around - ids: + first.append((f"({table}.uid IN (SELECT uid FROM role_places " + "WHERE place_id IN (SELECT value FROM json_each(?))) " + f"AND {table}.uid NOT IN (SELECT uid FROM role_places " + "WHERE place_id IS NOT NULL " + "AND place_id NOT IN (SELECT value FROM json_each(?))))", + [json.dumps(sorted(around - ids)), json.dumps(sorted(around))])) + clauses[:0] = [sql for sql, _ in first] + params[:0] = [value for _, values in first for value in values] if areas: remote.append(_areas_sql(areas, table)) for sql, values in remote: @@ -820,8 +859,9 @@ class Store: the way a board's own structured field is (docs/decisions.md, "a guess is not a board field"). So once a role has been through `extract` (`extracted_at` is set), this upsert leaves `function` - alone for good, whatever the plugin's raw guess now says — see - `set_extracted` for the other half of the rule. + alone, whatever the plugin's raw guess now says, until a better text + puts the role back in extract's queue (below) — see `set_extracted` + for the other half of the rule. The role's places (`role_places`) are the texts of `raw["places"]`, else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved @@ -833,9 +873,13 @@ class Store: under the same keep-filled rule. `teaser` is kept-filled too. A role is embedded from its description, else its teaser, else its - title (`app.jobs.embed.document`). When a pull brings a better one of - these than the role had, its vector is dropped so the next `embed` - reads it. + title (`app.jobs.embed.document`), and `extract` reads the same. When + a pull brings a better one of these than the role had, its vector is + dropped so the next `embed` reads it, and `extracted_at` and + `extract_tries` are reset so `extract` reads it again (if it is still + in a cut). That read overwrites the model's own columns and fills + what is still empty (`set_extracted`); until it comes, `function` is + the plugin's guess's again, as for any role not yet read. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -922,14 +966,17 @@ class Store: if before is not None: conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?", (row["uid"],)) - # `embed` encodes a role once, from the best text it has. When a - # later pull brings a better one, as a host that states the text - # on the role's own page does, the vector goes, so the next - # `embed` reads it. + # `embed` encodes a role once, from the best text it has, and + # `extract` reads it once. When a later pull brings a better one, + # as fetch text does for a host that states the text on the + # role's own page, the vector goes and the role waits for + # extract again, so both read the new text. if before is not None and _text_rank(row["description"] or before["description"], row["teaser"] or before["teaser"]) \ > _text_rank(before["description"], before["teaser"]): conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],)) + conn.execute("UPDATE roles SET extracted_at = NULL, extract_tries = 0 " + "WHERE uid=?", (row["uid"],)) return before is None def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None: @@ -1497,14 +1544,19 @@ class Store: return [g for g in found.gates if g.field == "country" and g.mode == "require"] \ if found is not None else [] - def tag_counts(self, agent_id: int | None = None) -> list[tuple[str, int, int]]: + def tag_counts(self, agent_id: int | None = None, + fetch_gate: bool = False) -> list[tuple[str, int, int]]: """`(tag, open_count, unread_count)` over open roles, most-used tag first and then alphabetically. What `/api/filters` serves; the companies `agent_id` hid are left out, as in `roles`, and with no - agent nothing is hidden.""" + agent nothing is hidden. With `fetch_gate`, only the roles the + agent's feed shows count (`_SHOWN`).""" where = "r.closed_at IS NULL AND r.dup_of IS NULL" params: list[Any] = [] - if agent_id is not None: + if agent_id is not None and fetch_gate: + where += f" AND {_SHOWN}" + params += [agent_id, agent_id] + elif agent_id is not None: where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies " "WHERE agent_id = ? AND affinity = -1)") params.append(agent_id) @@ -2317,7 +2369,11 @@ class Store: empty, match, values = _gate_sql(part, gaz) whens.append(f"WHEN {match} THEN {k}") params += values - columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}") + # A place gate's match goes first: an unresolved place can still + # meet one of its values (`_gate_sql`). + case = (f"{' '.join(whens)} WHEN {empty} THEN -1" if gate.field == "place" + else f"WHEN {empty} THEN -1 {' '.join(whens)}") + columns.append(f"CASE {case} ELSE 0 END AS g{n}") select = ", ".join(["r.uid", "r.posted_at", "COALESCE(ac.affinity, 0) AS affinity", "COALESCE(c.all_countries, 0) AS all_countries", @@ -2596,25 +2652,30 @@ class Store: "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")} - def open_companies(self, agent_id: int | None = None + def open_companies(self, agent_id: int | None = None, fetch_gate: bool = False ) -> list[tuple[str, list[str], int, int]]: """`(company_key, names, open roles, affinity)` for every company with an open role, what the Company field suggests from. `names` are the company row's name and then every spelling its roles carry, commonest first, so a search finds a company by any of them. A company `agent_id` hid is left out; `affinity` is that agent's, 0 without - one. Copies are not counted, as in the feed.""" + one. Copies are not counted, as in the feed. With `fetch_gate`, only + the roles the agent's feed shows count (`_SHOWN`), and a company with + none of them is left out.""" + gate, params = "", [agent_id] + if agent_id is not None and fetch_gate: + gate, params = f"AND {_SHOWN}", [agent_id, agent_id, agent_id] rows = self._rows( - """SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, - c.name AS own, COALESCE(ac.affinity, 0) AS affinity - FROM roles r - LEFT JOIN companies c ON c.key = r.company_key - LEFT JOIN agent_companies ac ON ac.company_key = r.company_key - AND ac.agent_id = ? - WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' - AND COALESCE(ac.affinity, 0) != -1 - GROUP BY r.company_key, r.company - ORDER BY r.company_key, n DESC, r.company""", (agent_id,)) + f"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, + c.name AS own, COALESCE(ac.affinity, 0) AS affinity + FROM roles r + LEFT JOIN companies c ON c.key = r.company_key + LEFT JOIN agent_companies ac ON ac.company_key = r.company_key + AND ac.agent_id = ? + WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' + AND COALESCE(ac.affinity, 0) != -1 {gate} + GROUP BY r.company_key, r.company + ORDER BY r.company_key, n DESC, r.company""", params) found: dict[str, tuple[list[str], list[int], int]] = {} for r in rows: names, counts, _ = found.setdefault( @@ -2625,24 +2686,31 @@ class Store: return [(key, names, sum(counts), affinity) for key, (names, counts, affinity) in found.items()] - def open_places(self) -> list[tuple[str, str, int]]: + def open_places(self, agent_id: int | None = None) -> list[tuple[str, str, int]]: """`(place, country, open roles)` for every place line an open role carries, what `places.suggest` reads. Copies are not counted, as in - the feed.""" + the feed. Given `agent_id`, only the roles that agent's feed shows + (`_SHOWN`).""" + gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \ + else ("", []) return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows( - "SELECT place, country, COUNT(*) AS n FROM roles " - "WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' " - "GROUP BY place, country")] + "SELECT r.place, r.country, COUNT(*) AS n FROM roles r " + f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.place != '' {gate} " + "GROUP BY r.place, r.country", params)] - def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]: + def open_remote_areas(self, agent_id: int | None = None + ) -> list[tuple[tuple[int, ...], int]]: """`(area, open roles)` for every remote area an open remote role holds, the area as place ids in order: what `places.suggest` counts - the remote tokens from. Copies are not counted, as in the feed.""" + the remote tokens from. Copies are not counted, as in the feed. + Given `agent_id`, only the roles that agent's feed shows (`_SHOWN`).""" + gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \ + else ("", []) found: dict[str, list[int]] = {} for r in self._rows( "SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid " - "WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' " - "ORDER BY rr.uid, rr.rowid"): + f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' {gate} " + "ORDER BY rr.uid, rr.rowid", params): found.setdefault(str(r[0]), []).append(int(r[1])) return list(Counter(tuple(ids) for ids in found.values()).items()) diff --git a/app/web/api.py b/app/web/api.py index 295db98..39c2bcf 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -294,10 +294,12 @@ class FiltersOut(BaseModel): def list_filters(store: StoreDep, agent: int | None = None) -> FiltersOut: """The tag vocabulary for the feed's filter row, most-used first. The companies `agent` hid (the default agent's without one) are left out, - matching `/api/roles`.""" + matching `/api/roles`; with `agent` named, so is every role its fetch + gate rejects, so a count is what the feed lists for the tag.""" hidden_for = _agent_id(store, agent) return FiltersOut(tags=[TagCount(tag=tag, count=count, unread=unread) - for tag, count, unread in store.tag_counts(hidden_for)]) + for tag, count, unread in store.tag_counts( + hidden_for, fetch_gate=agent is not None)]) # ------------------------------------------------------------ suggestions @@ -343,14 +345,17 @@ def suggest_companies( folded as places are (`places.fold`), so `orsted` finds Ørsted and `kobenhavn` finds a name with København in it. By open roles, most first. The companies `agent` (the default agent when absent) hid are - left out, as the feed leaves them out. An empty `q` suggests nothing.""" + left out, as the feed leaves them out; with `agent` named, the count is + the roles its feed shows, past its fetch gate. An empty `q` suggests + nothing.""" text = places.fold(q) if not text: return [] aliases = store.company_aliases() + companies = store.open_companies(_agent_id(store, agent), fetch_gate=agent is not None) found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count, affinity=affinity) - for key, names, count, affinity in store.open_companies(_agent_id(store, agent)) + for key, names, count, affinity in companies if any(_starts_a_word(text, places.fold(name)) for name in names) or any(_starts_a_word(text, fold) for fold in aliases.get(key, ()))] found.sort(key=lambda c: (-c.count, c.name.casefold())) @@ -370,11 +375,13 @@ def suggest_places( open role's place line and every open remote role's area). The count of `remote:any` is the remote roles in the scope of `agent`, the default agent when absent, as the feed counts them, and the other remote tokens - are kept to that agent's scope, its `require` gates on country. An empty - `q` suggests nothing.""" + are kept to that agent's scope, its `require` gates on country. With + `agent` named, every count is of the roles its feed shows, past its + fetch gate. An empty `q` suggests nothing.""" if not places.fold(q): return [] viewer = _agent_id(store, agent) + shown = viewer if agent is not None else None gaz = store.gazetteer() found = store.agent(viewer) if viewer is not None else None scope = {p for g in (found.gates if found is not None else ()) @@ -382,10 +389,11 @@ def suggest_places( for v in g.values for p in gaz.lookup(f"country:{v}") or ()} return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count) for s in places.suggest( - q, store.open_places(), limit, remote=store.open_remote_areas(), + q, store.open_places(shown), limit, remote=store.open_remote_areas(shown), gaz=gaz, scope=scope, remote_any=lambda: store.count_roles(places=["remote:any"], agent=viewer, - by_score=False))] + by_score=False, + fetch_gate=agent is not None))] @router.get("/places/labels") @@ -772,7 +780,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the default agent. A key with roles and no company row gets one first (see `Store.set_affinity`); 404 for a key with neither. A change here is not - seen in Agent match until `score` has run, so it queues one + seen in "What I'm looking for" until `score` has run, so it queues one (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") diff --git a/tests/test_extract.py b/tests/test_extract.py index 951213a..b6062d0 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -130,6 +130,46 @@ def test_the_next_upsert_keeps_what_extract_filled( assert role is not None and (role.country, role.seniority) == ("DE", "mid") +def test_a_role_read_from_its_title_is_read_again_once_its_text_arrives( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + """Fetch text stores what a page filled through `upsert_role`: a better + text than the role was read from puts it back in extract's queue, its + tries forgiven, and a pull that brings no better text does not.""" + bare = posting("1") + bare.body = "" + upsert(store, bare) + ranked(store, bare) + stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="mid", function="sales") + for r in roles]) + modeltask.run(extract.TASK, config, store) + with store.transaction() as conn: + conn.execute("UPDATE roles SET extract_tries = 2 WHERE source_id = '1'") + + upsert(store, bare) + assert store.extract_pending() == 0, "the same text again is no reason to read it" + + upsert(store, posting("1", function="engineering")) + + role = store.role(bare.uid) + assert role is not None and role.extracted_at is None + assert role.function == "sales", "the pull that brings the text keeps extract's function" + assert [r.uid for r in store.extract_waiting(10)] == [bare.uid] + with store.transaction() as conn: + assert conn.execute("SELECT extract_tries FROM roles WHERE uid = ?", + (bare.uid,)).fetchone()[0] == 0 + model = stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="senior") + for r in roles]) + modeltask.run(extract.TASK, config, store) + [[sent]] = model.asked + assert sent["text"] == "We pay 80.000-95.000 EUR. Hybrid in Berlin." + role = store.role(bare.uid) + assert role is not None and role.extracted_at is not None + assert (role.seniority_model, role.seniority) == ("senior", "senior") + + upsert(store, posting("1")) + assert store.extract_pending() == 0, "a text no better than before is read once" + + def test_closed_and_answered_roles_are_not_offered( store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: for sid in ("open", "closed", "done"): diff --git a/tests/test_fetch_text.py b/tests/test_fetch_text.py index aeba2e6..fe9b77d 100644 --- a/tests/test_fetch_text.py +++ b/tests/test_fetch_text.py @@ -237,6 +237,24 @@ def test_a_batch_stores_what_the_pages_filled_and_drops_those_vectors( assert again.place == "Copenhagen" +def test_a_role_extract_read_without_text_waits_for_it_again_once_its_page_is_read( + store: Store, tmp_path: Path, web: dict[str, Any]) -> None: + """Extract read the title alone; the page's text puts the role back in + its queue, and a page that gave nothing leaves it read.""" + board(store, "fake:acme") + filled, empty = (role(store, "fake:acme", n) for n in ("1", "2")) + with store.transaction() as conn: + conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', " + "extract_tries = 1") + web["answers"].update({f"{BASE}/acme/1": b"Python in full"}) + + fetch_text.run_batch(config(tmp_path), store, target="fake:acme", uids=[filled, empty]) + + first, second = store.role(filled), store.role(empty) + assert first is not None and first.extracted_at is None + assert second is not None and second.extracted_at == "2026-09-22T00:00:00+00:00" + + def test_a_board_switched_off_since_the_plan_reads_nothing( store: Store, tmp_path: Path, web: dict[str, Any]) -> None: sub = board(store, "fake:acme") diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 0537e6c..f166de4 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -343,8 +343,9 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store, several: dict[str, str]) -> None: found = gates(store, Gate(field="place", values=["Malmö", "Lund", "oberkochen"], mode="any")) + # "Remote (SE)" is placed at Sweden, which holds Malmö: no evidence against. assert {uid for uid, g in found.items() if g == 1} == { - several["malmo"], several["lund"], several["zeiss"]} + several["malmo"], several["lund"], several["zeiss"], several["remote"]} # ------------------------------------------------------------ unresolved texts diff --git a/tests/test_judge.py b/tests/test_judge.py index 077f792..48e5208 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -194,12 +194,28 @@ def hashlib_old_key(agent: Any, text: str) -> str: return hashlib.sha1(blob.encode()).hexdigest()[:16] -def test_only_enabled_agents_with_a_cv_and_a_prompt_are_judged(store: Store) -> None: +def test_only_enabled_agents_with_a_cv_are_judged(store: Store) -> None: for name, kwargs in (("No CV", {"cv": " "}), ("Off", {"enabled": False})): ranked(store, hunter(store, name, **kwargs), name) bare = store.save_agent("No prompt", cv_text=CV) - ranked(store, bare, "bare") - assert judge.waiting(store) == [] + [uid] = ranked(store, bare, "bare") + assert [(a.id, row["uid"]) for a, row, _, _ in judge.waiting(store)] == [(bare.id, uid)] + + +def test_an_agent_without_a_prompt_is_judged_without_wants( + store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None: + """A CV alone is enough: the cut is the CV list, and the profile has no + `` block.""" + bare = store.save_agent("No prompt", cv_text=CV) + [uid] = ranked(store, bare, "1") + model = stub(monkeypatch) + + modeltask.run(judge.TASK, config, store) + + found = store.judgement(bare.id, uid) + assert found is not None and (found.skills, found.experience, found.career) == (8, 7, 9) + [system] = model.systems + assert CV in system and " None: diff --git a/tests/test_remote.py b/tests/test_remote.py index f35cfb9..cbac403 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -15,7 +15,7 @@ import pytest from desk.models import Item from app import places, prep -from app.jobs import extract, reach +from app.jobs import extract, reach, score from app.models import Agent, Gate from app.places import Gazetteer, label, known, parse, seed, suggest from app.prep import words @@ -468,6 +468,10 @@ def test_the_api_suggests_labels_and_filters_the_remote_tokens(tmp_path: Path) - prep.run(store, dupes=False) agent = store.save_agent("Europe", "python backend", [ Gate(field="country", values=EUROPE, mode="require")]) + # Scored as `score` would, since a named agent's counts are its feed's. + store.replace_scores(agent.id, [ + (c.uid, 0.5, ", ".join(score.failed_gates(c, agent.gates))) + for c in store.score_candidates(agent.id, agent.gates)]) with TestClient(app) as client: found = client.get("/api/suggest/places", params={"q": "remote"}).json() assert found[0] == {"token": "remote:any", "label": "Remote", "kind": "remote", diff --git a/tests/test_score.py b/tests/test_score.py index adc081d..7f00b18 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -346,9 +346,113 @@ def test_any_gates_are_one_group_a_role_must_match_one_of( assert scores[copenhagen][1] == "" and scores[remote_de][1] == "" assert scores[onsite_de] == (0.0, "none of country DK, remote remote") assert scores[blank][1] == "" - # "Hybrid" is a country-less line the remote lookup reads as hybrid, so - # the group has evidence against remote and none for country: gated. - assert scores[hybrid_nowhere][1].startswith("none of") + # "Hybrid" is a country-less line the remote lookup reads as hybrid: the + # role may still be in Denmark, so it passes until extract reads it. + assert scores[hybrid_nowhere][1] == "" + + +def test_an_unknown_location_passes_the_any_group_whatever_its_remote( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """A place line the gazetteer cannot resolve is no evidence the role is + elsewhere, for the country and for the place gate: the role passes, + hybrid or not. One whose unknown line names a gate value by whole words + matches it, and a role in a resolved place elsewhere is still gated, its + remote unknown or not.""" + jobbank = role(store, "1", place="Storkøbenhavn; Udlandet (øvrige)", remote="hybrid") + office = role(store, "2", place="Fleet Place Office", remote="onsite") + avedore = role(store, "3", place="Avedøre Holme") + berlin = role(store, "4", place="Berlin, Germany") + berlin_hybrid = role(store, "5", place="Berlin, Germany", remote="hybrid") + agent = store.save_agent("Malmö, Avedøre or remote", "", [ + Gate(field="country", values=["SE"], mode="any"), + Gate(field="remote", values=["remote"], mode="any"), + Gate(field="place", values=["Malmö", "Avedøre Holme"], mode="any")]) + + run(config, store, score.run_score) + + states = {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} + assert states[jobbank] == [-1, 0, -1] and states[office] == [-1, 0, -1] + assert states[avedore] == [-1, -1, 1] + assert states[berlin] == [0, -1, 0] + scores = store.scores(agent.id) + assert all(scores[uid][1] == "" for uid in (jobbank, office, avedore)) + reason = "none of country SE, remote remote, place Malmö / Avedøre Holme" + assert scores[berlin] == (0.0, reason) and scores[berlin_hybrid] == (0.0, reason) + + +def test_a_remote_only_group_still_gates_a_hybrid_role( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """With no country or place in the group, remote is the only evidence: + a hybrid role is gated, one whose remote is unknown passes.""" + hybrid = role(store, "1", place="Headquarters", remote="hybrid") + unknown = role(store, "2", place="Headquarters") + agent = store.save_agent("Remote", "", [Gate(field="remote", values=["remote"], mode="any")]) + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + assert scores[hybrid] == (0.0, "none of remote remote") + assert scores[unknown][1] == "" + + +def test_a_region_that_holds_the_agents_places_is_no_evidence_against( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """Øresund holds the Capital Region and Skåne, so a role placed only + there meets a gate on Denmark and one on Copenhagen; Berlin meets + neither. The feed's Place filter still reads only what lies under.""" + oresund = role(store, "1", place="Øresundsregionen", remote="onsite") + berlin = role(store, "2", place="Berlin, Germany", remote="onsite") + by_country = store.save_agent("Denmark", "", [ + Gate(field="country", values=["DK"], mode="any")]) + by_town = store.save_agent("Copenhagen", "", [ + Gate(field="place", values=["copenhagen"], mode="any")]) + world = store.save_agent("Denmark only", "", [Gate(field="country", values=["DK"])]) + + run(config, store, score.run_score) + + for agent in (by_country, by_town, world): + scores = store.scores(agent.id) + assert scores[oresund][1] == "" + assert scores[berlin][1] != "" + assert oresund not in {r.uid for r in store.roles(places=["country:DK"])} + + +def test_a_wider_place_beside_a_place_elsewhere_is_no_match( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """Europe alone is no evidence against Denmark; `London; Europe` is in + London, and `Aarhus; Denmark` is not in Copenhagen.""" + europe = role(store, "1", place="Europe", remote="onsite") + london = role(store, "2", place="London; Europe", remote="onsite") + aarhus = role(store, "3", place="Aarhus; Denmark", remote="onsite") + denmark = role(store, "4", place="Denmark", remote="onsite") + by_country = store.save_agent("Denmark", "", [ + Gate(field="country", values=["DK"], mode="any")]) + by_town = store.save_agent("Copenhagen", "", [ + Gate(field="place", values=["copenhagen"], mode="any")]) + + run(config, store, score.run_score) + + scores = store.scores(by_country.id) + assert scores[europe][1] == "" and scores[aarhus][1] == "" and scores[denmark][1] == "" + assert scores[london] == (0.0, "none of country DK") + scores = store.scores(by_town.id) + assert scores[europe][1] == "" and scores[denmark][1] == "" + assert scores[london] == (0.0, "none of place copenhagen") + assert scores[aarhus] == (0.0, "none of place copenhagen") + + +def test_a_country_that_holds_the_agents_town_is_no_match_for_a_role_placed_elsewhere( + store: Store, config: Config, fake: FakeEmbedder) -> None: + """Sweden holds Malmö, yet a role in Stockholm is evidence against it.""" + stockholm = role(store, "1", place="Stockholm, Sweden", remote="onsite") + malmo = role(store, "2", place="Malmö, Sweden", remote="onsite") + agent = store.save_agent("Malmö", "", [Gate(field="place", values=["Malmö"], mode="any")]) + + run(config, store, score.run_score) + + scores = store.scores(agent.id) + assert scores[stockholm] == (0.0, "none of place Malmö") + assert scores[malmo][1] == "" def test_failed_gates_reads_evidence_only() -> None: @@ -365,6 +469,24 @@ def test_failed_gates_reads_evidence_only() -> None: "none of remote remote, seniority senior"] +def test_an_any_group_with_every_place_member_empty_does_not_fail() -> None: + """Remote is no evidence against a role whose location is unknown; a + known location elsewhere is, also with remote unknown.""" + gates = [Gate(field="country", values=["DK"], mode="any"), + Gate(field="remote", values=["remote"], mode="any"), + Gate(field="place", values=["Malmö"], mode="any")] + def cand(*states: int) -> Candidate: + return Candidate(uid="u", posted_at=NOW.isoformat(), gates=list(states)) + assert score.failed_gates(cand(-1, 0, -1), gates) == [] + assert score.failed_gates(cand(0, -1, -1), gates) == [ + "none of country DK, remote remote, place Malmö"] + assert score.failed_gates(cand(-1, -1, 0), gates) == [ + "none of country DK, remote remote, place Malmö"] + remote_only = [Gate(field="remote", values=["remote"], mode="any")] + assert score.failed_gates(cand(0), remote_only) == ["none of remote remote"] + assert score.failed_gates(cand(-1), remote_only) == [] + + def test_a_gate_is_require_any_or_exclude_and_has_no_weight() -> None: """Prefer gates left the agent: hybrid is a feed filter now.""" with pytest.raises(ValidationError): diff --git a/tests/test_web.py b/tests/test_web.py index a516fa4..5ca8a31 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -219,11 +219,44 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( assert [c["name"] for c in suggest("big da")] == ["Big Data Corp"] assert [c["name"] for c in suggest("data", limit=1)] == ["Big Data Corp"] assert suggest("") == [] - # Another agent hid nothing. + # Another agent hid nothing; named, it counts what its feed shows. other = store.save_agent("B") + store.replace_scores(other.id, [(r.uid, 0.5, "") for r in store.roles()]) assert "Hidden Data" in [c["name"] for c in suggest("data", agent=other.id)] +def test_a_named_agents_rail_counts_are_what_its_feed_shows( + client: TestClient, store: Store) -> None: + """Tag, company and place counts, "Remote" included, cover the roles the + feed lists for a named agent: past its fetch gate, of a company it has + not hidden. Without one they count as before.""" + for sid in "1234": + upsert(store, posting(sid, company="Gone" if sid == "4" else "Acme", + place="Malmö", remote="remote")) + uid = {r.source_id: r.uid for r in store.roles()} + for one in uid.values(): + store.set_tags(one, ["python"]) + store.save_agent("Default") + agent = store.save_agent("A") + store.replace_scores(agent.id, [(uid["1"], 0.5, ""), (uid["2"], 0.4, ""), + (uid["3"], 0.0, "title sales"), (uid["4"], 0.3, "")]) + store.set_affinity(agent.id, "gone", -1) + + def counts(**params: Any) -> tuple[int, int, int, int]: + tags = client.get("/api/filters", params=params).json()["tags"] + [acme] = client.get("/api/suggest/companies", params={"q": "acme", **params}).json() + malmo = client.get("/api/suggest/places", params={"q": "malm", **params}).json() + remote = client.get("/api/suggest/places", params={"q": "remote", **params}).json() + return (tags[0]["count"], acme["count"], + next(s["count"] for s in malmo if s["token"] == "place:malmo"), + next(s["count"] for s in remote if s["token"] == "remote:any")) + + feed = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json() + assert feed["total"] == 2 + assert counts(agent=agent.id) == (2, 2, 2, 2) + assert counts() == (4, 3, 4, 4) + + def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None: for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"), ("4", "København K"), ("5", "Kolding")):