diff --git a/app/storage/store.py b/app/storage/store.py index 4b1b01d..5099ff9 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -84,6 +84,15 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe _IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id " "AND a.enabled = 1 WHERE s.cut <> '')") +#: A role one agent's feed shows, as a `WHERE` on `r` taking the agent's id +#: twice: its fetch gate let the role through (`_roles_from` with +#: `fetch_gate`), and the company is not one it hid. With the open and kept +#: copy test, what the rail's counts for that agent cover. +_SHOWN = ("EXISTS (SELECT 1 FROM scores s WHERE s.uid = r.uid AND s.agent_id = ? " + "AND s.gated_reason = '') " + "AND r.company_key NOT IN (SELECT company_key FROM agent_companies " + "WHERE agent_id = ? AND affinity = -1)") + #: The order within one agent's cut, over its scores as `s`: in by fit, best #: fit first, then in by CV closeness, closest first, then kept. _CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, " @@ -1521,14 +1530,19 @@ class Store: return [g for g in found.gates if g.field == "country" and g.mode == "require"] \ if found is not None else [] - def tag_counts(self, agent_id: int | None = None) -> list[tuple[str, int, int]]: + def tag_counts(self, agent_id: int | None = None, + fetch_gate: bool = False) -> list[tuple[str, int, int]]: """`(tag, open_count, unread_count)` over open roles, most-used tag first and then alphabetically. What `/api/filters` serves; the companies `agent_id` hid are left out, as in `roles`, and with no - agent nothing is hidden.""" + agent nothing is hidden. With `fetch_gate`, only the roles the + agent's feed shows count (`_SHOWN`).""" where = "r.closed_at IS NULL AND r.dup_of IS NULL" params: list[Any] = [] - if agent_id is not None: + if agent_id is not None and fetch_gate: + where += f" AND {_SHOWN}" + params += [agent_id, agent_id] + elif agent_id is not None: where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies " "WHERE agent_id = ? AND affinity = -1)") params.append(agent_id) @@ -2624,25 +2638,30 @@ class Store: "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")} - def open_companies(self, agent_id: int | None = None + def open_companies(self, agent_id: int | None = None, fetch_gate: bool = False ) -> list[tuple[str, list[str], int, int]]: """`(company_key, names, open roles, affinity)` for every company with an open role, what the Company field suggests from. `names` are the company row's name and then every spelling its roles carry, commonest first, so a search finds a company by any of them. A company `agent_id` hid is left out; `affinity` is that agent's, 0 without - one. Copies are not counted, as in the feed.""" + one. Copies are not counted, as in the feed. With `fetch_gate`, only + the roles the agent's feed shows count (`_SHOWN`), and a company with + none of them is left out.""" + gate, params = "", [agent_id] + if agent_id is not None and fetch_gate: + gate, params = f"AND {_SHOWN}", [agent_id, agent_id, agent_id] rows = self._rows( - """SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, - c.name AS own, COALESCE(ac.affinity, 0) AS affinity - FROM roles r - LEFT JOIN companies c ON c.key = r.company_key - LEFT JOIN agent_companies ac ON ac.company_key = r.company_key - AND ac.agent_id = ? - WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' - AND COALESCE(ac.affinity, 0) != -1 - GROUP BY r.company_key, r.company - ORDER BY r.company_key, n DESC, r.company""", (agent_id,)) + f"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n, + c.name AS own, COALESCE(ac.affinity, 0) AS affinity + FROM roles r + LEFT JOIN companies c ON c.key = r.company_key + LEFT JOIN agent_companies ac ON ac.company_key = r.company_key + AND ac.agent_id = ? + WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != '' + AND COALESCE(ac.affinity, 0) != -1 {gate} + GROUP BY r.company_key, r.company + ORDER BY r.company_key, n DESC, r.company""", params) found: dict[str, tuple[list[str], list[int], int]] = {} for r in rows: names, counts, _ = found.setdefault( @@ -2653,24 +2672,31 @@ class Store: return [(key, names, sum(counts), affinity) for key, (names, counts, affinity) in found.items()] - def open_places(self) -> list[tuple[str, str, int]]: + def open_places(self, agent_id: int | None = None) -> list[tuple[str, str, int]]: """`(place, country, open roles)` for every place line an open role carries, what `places.suggest` reads. Copies are not counted, as in - the feed.""" + the feed. Given `agent_id`, only the roles that agent's feed shows + (`_SHOWN`).""" + gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \ + else ("", []) return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows( - "SELECT place, country, COUNT(*) AS n FROM roles " - "WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' " - "GROUP BY place, country")] + "SELECT r.place, r.country, COUNT(*) AS n FROM roles r " + f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.place != '' {gate} " + "GROUP BY r.place, r.country", params)] - def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]: + def open_remote_areas(self, agent_id: int | None = None + ) -> list[tuple[tuple[int, ...], int]]: """`(area, open roles)` for every remote area an open remote role holds, the area as place ids in order: what `places.suggest` counts - the remote tokens from. Copies are not counted, as in the feed.""" + the remote tokens from. Copies are not counted, as in the feed. + Given `agent_id`, only the roles that agent's feed shows (`_SHOWN`).""" + gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \ + else ("", []) found: dict[str, list[int]] = {} for r in self._rows( "SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid " - "WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' " - "ORDER BY rr.uid, rr.rowid"): + f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' {gate} " + "ORDER BY rr.uid, rr.rowid", params): found.setdefault(str(r[0]), []).append(int(r[1])) return list(Counter(tuple(ids) for ids in found.values()).items()) diff --git a/app/web/api.py b/app/web/api.py index 295db98..39c2bcf 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -294,10 +294,12 @@ class FiltersOut(BaseModel): def list_filters(store: StoreDep, agent: int | None = None) -> FiltersOut: """The tag vocabulary for the feed's filter row, most-used first. The companies `agent` hid (the default agent's without one) are left out, - matching `/api/roles`.""" + matching `/api/roles`; with `agent` named, so is every role its fetch + gate rejects, so a count is what the feed lists for the tag.""" hidden_for = _agent_id(store, agent) return FiltersOut(tags=[TagCount(tag=tag, count=count, unread=unread) - for tag, count, unread in store.tag_counts(hidden_for)]) + for tag, count, unread in store.tag_counts( + hidden_for, fetch_gate=agent is not None)]) # ------------------------------------------------------------ suggestions @@ -343,14 +345,17 @@ def suggest_companies( folded as places are (`places.fold`), so `orsted` finds Ørsted and `kobenhavn` finds a name with København in it. By open roles, most first. The companies `agent` (the default agent when absent) hid are - left out, as the feed leaves them out. An empty `q` suggests nothing.""" + left out, as the feed leaves them out; with `agent` named, the count is + the roles its feed shows, past its fetch gate. An empty `q` suggests + nothing.""" text = places.fold(q) if not text: return [] aliases = store.company_aliases() + companies = store.open_companies(_agent_id(store, agent), fetch_gate=agent is not None) found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count, affinity=affinity) - for key, names, count, affinity in store.open_companies(_agent_id(store, agent)) + for key, names, count, affinity in companies if any(_starts_a_word(text, places.fold(name)) for name in names) or any(_starts_a_word(text, fold) for fold in aliases.get(key, ()))] found.sort(key=lambda c: (-c.count, c.name.casefold())) @@ -370,11 +375,13 @@ def suggest_places( open role's place line and every open remote role's area). The count of `remote:any` is the remote roles in the scope of `agent`, the default agent when absent, as the feed counts them, and the other remote tokens - are kept to that agent's scope, its `require` gates on country. An empty - `q` suggests nothing.""" + are kept to that agent's scope, its `require` gates on country. With + `agent` named, every count is of the roles its feed shows, past its + fetch gate. An empty `q` suggests nothing.""" if not places.fold(q): return [] viewer = _agent_id(store, agent) + shown = viewer if agent is not None else None gaz = store.gazetteer() found = store.agent(viewer) if viewer is not None else None scope = {p for g in (found.gates if found is not None else ()) @@ -382,10 +389,11 @@ def suggest_places( for v in g.values for p in gaz.lookup(f"country:{v}") or ()} return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count) for s in places.suggest( - q, store.open_places(), limit, remote=store.open_remote_areas(), + q, store.open_places(shown), limit, remote=store.open_remote_areas(shown), gaz=gaz, scope=scope, remote_any=lambda: store.count_roles(places=["remote:any"], agent=viewer, - by_score=False))] + by_score=False, + fetch_gate=agent is not None))] @router.get("/places/labels") @@ -772,7 +780,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the default agent. A key with roles and no company row gets one first (see `Store.set_affinity`); 404 for a key with neither. A change here is not - seen in Agent match until `score` has run, so it queues one + seen in "What I'm looking for" until `score` has run, so it queues one (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") diff --git a/tests/test_remote.py b/tests/test_remote.py index b888b21..90c1296 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -15,7 +15,7 @@ import pytest from desk.models import Item from app import places, prep -from app.jobs import extract, reach +from app.jobs import extract, reach, score from app.models import Agent, Gate from app.places import Gazetteer, label, known, parse, seed, suggest from app.prep import words @@ -468,6 +468,10 @@ def test_the_api_suggests_labels_and_filters_the_remote_tokens(tmp_path: Path) - prep.run(store, dupes=False) agent = store.save_agent("Europe", "python backend", [ Gate(field="country", values=EUROPE, mode="require")]) + # Scored as `score` would, since a named agent's counts are its feed's. + store.replace_scores(agent.id, [ + (c.uid, 0.5, ", ".join(score.failed_gates(c, agent.gates))) + for c in store.score_candidates(agent.id, agent.gates)]) with TestClient(app) as client: found = client.get("/api/suggest/places", params={"q": "remote"}).json() assert found[0] == {"token": "remote:any", "label": "Remote", "kind": "remote", diff --git a/tests/test_web.py b/tests/test_web.py index a516fa4..5ca8a31 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -219,11 +219,44 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden( assert [c["name"] for c in suggest("big da")] == ["Big Data Corp"] assert [c["name"] for c in suggest("data", limit=1)] == ["Big Data Corp"] assert suggest("") == [] - # Another agent hid nothing. + # Another agent hid nothing; named, it counts what its feed shows. other = store.save_agent("B") + store.replace_scores(other.id, [(r.uid, 0.5, "") for r in store.roles()]) assert "Hidden Data" in [c["name"] for c in suggest("data", agent=other.id)] +def test_a_named_agents_rail_counts_are_what_its_feed_shows( + client: TestClient, store: Store) -> None: + """Tag, company and place counts, "Remote" included, cover the roles the + feed lists for a named agent: past its fetch gate, of a company it has + not hidden. Without one they count as before.""" + for sid in "1234": + upsert(store, posting(sid, company="Gone" if sid == "4" else "Acme", + place="Malmö", remote="remote")) + uid = {r.source_id: r.uid for r in store.roles()} + for one in uid.values(): + store.set_tags(one, ["python"]) + store.save_agent("Default") + agent = store.save_agent("A") + store.replace_scores(agent.id, [(uid["1"], 0.5, ""), (uid["2"], 0.4, ""), + (uid["3"], 0.0, "title sales"), (uid["4"], 0.3, "")]) + store.set_affinity(agent.id, "gone", -1) + + def counts(**params: Any) -> tuple[int, int, int, int]: + tags = client.get("/api/filters", params=params).json()["tags"] + [acme] = client.get("/api/suggest/companies", params={"q": "acme", **params}).json() + malmo = client.get("/api/suggest/places", params={"q": "malm", **params}).json() + remote = client.get("/api/suggest/places", params={"q": "remote", **params}).json() + return (tags[0]["count"], acme["count"], + next(s["count"] for s in malmo if s["token"] == "place:malmo"), + next(s["count"] for s in remote if s["token"] == "remote:any")) + + feed = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json() + assert feed["total"] == 2 + assert counts(agent=agent.id) == (2, 2, 2, 2) + assert counts() == (4, 3, 4, 4) + + def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None: for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"), ("4", "København K"), ("5", "Kolding")):