From 57cac5f8966c0517128e9303c74d65be8fc0b991 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 16:00:06 +0200 Subject: [PATCH] Add migration 27 and read a remote role's area in the chips, the gates and the rows roles.remote_model, roles.remote_where_model and role_remote, with a stored remote no host word gives kept as the host's word. The remote tokens filter over role_remote, remote_from and a require gate on country read a remote role through its area, extract waits while remote_where_model is NULL and writes the model's remote outright, and rows carry remote_where. Co-Authored-By: Claude Opus 5.5 --- app/models.py | 20 +++- app/storage/schema.py | 62 ++++++++++- app/storage/store.py | 214 +++++++++++++++++++++++++++++------- tests/test_companies.py | 4 +- tests/test_dupes_storage.py | 2 +- tests/test_fetch_terms.py | 2 +- tests/test_overrides.py | 2 +- tests/test_storage.py | 18 +-- 8 files changed, 262 insertions(+), 62 deletions(-) diff --git a/app/models.py b/app/models.py index 5641a6b..ea971e7 100644 --- a/app/models.py +++ b/app/models.py @@ -970,6 +970,10 @@ class Role(Model): #: Every place of the role, in the host's order. Not a column: filled by #: `Store.role` from `role_places`. places: list[RolePlace] = [] + #: Where a remote hire may live, as a label: "anywhere", "Europe", + #: "United States, Canada". '' when the role is not remote or its area is + #: not stated. Not a column: filled by `Store` from `role_remote`. + remote_where: str = "" @computed_field # type: ignore[prop-decorator] @property @@ -1076,6 +1080,9 @@ class RoleRow(Model): excerpt: str = "" #: How many open roles are copies of this one (`Role.copies`), 0 when none. copies: int = 0 + #: As `Role.remote_where`: the area a remote hire may live in, '' when + #: the role is not remote or does not say. + remote_where: str = "" @computed_field # type: ignore[prop-decorator] @property @@ -1167,11 +1174,14 @@ class Application(Model): #: The role columns a gate can name. `company` matches on the company key; #: `place` takes the Place filter's tokens (`app.places`: `country:DK`, -#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are -#: case-insensitive equality, except `salary_min` (at least the value) and -#: `languages` (every language the role requires is one of the values). -GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min", - "currency", "permit", "languages", "company"] +#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; +#: `remote_from` takes place tokens (`country:DK`, `country:EU`, `anywhere`) +#: and is met by a remote role whose area overlaps one, as a `remote:` chip +#: is; the rest are case-insensitive equality, except `salary_min` (at least +#: the value) and `languages` (every language the role requires is one of +#: the values). +GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function", + "salary_min", "currency", "permit", "languages", "company"] GATE_FIELDS: tuple[str, ...] = get_args(GateField) GateMode = Literal["require", "any", "prefer"] diff --git a/app/storage/schema.py b/app/storage/schema.py index eb3fa72..a3cc164 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -118,7 +118,9 @@ CREATE TABLE IF NOT EXISTS roles ( remote_host TEXT NOT NULL DEFAULT '', country_host TEXT NOT NULL DEFAULT '', employment TEXT NOT NULL DEFAULT '', - prep_due INTEGER NOT NULL DEFAULT 0 + prep_due INTEGER NOT NULL DEFAULT 0, + remote_model TEXT NOT NULL DEFAULT '', + remote_where_model TEXT ); -- The *_host columns hold the host's own words as a reader handed them on -- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps @@ -129,9 +131,12 @@ CREATE TABLE IF NOT EXISTS roles ( -- arrives and embedded while there is none; a later empty teaser does not -- wipe it. seniority_model, seniority_stated and seniority_title are the -- three levels behind roles.seniority: the model's (extract), the host's own --- word mapped to app.seniority's vocabulary, and the title's. No comment sits --- among the last columns: SQLite cannot drop a column a comment precedes, --- which the migration tests do. +-- word mapped to app.seniority's vocabulary, and the title's. remote_model is +-- extract's remote, which wins over the host's word and the place line, and +-- remote_where_model the names it read for where a remote hire may live (a +-- JSON list, NULL never asked, [] not stated). No comment sits among the last +-- columns: SQLite cannot drop a column a comment precedes, which the +-- migration tests do. -- The gazetteer: a place is a continent, a country (key the ISO code), a -- region or a city, in the place parent_id names. key is unique per kind: @@ -177,6 +182,15 @@ CREATE TABLE IF NOT EXISTS role_places ( url TEXT NOT NULL DEFAULT '' ); +-- A remote role's area, the places a remote hire may live in (app/prep/area.py), +-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with +-- no row is not remote, or remote with its area not stated. +CREATE TABLE IF NOT EXISTS role_remote ( + uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + PRIMARY KEY (uid, place_id) +); + -- The open vocabulary summarise reads off a posting: the role family, the -- technologies and skills it names, and the domain. What the feed's filter -- row is built from; see Store.set_tags and Store.tag_counts. @@ -424,6 +438,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid); CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid); CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id); +-- The remote roles whose area holds a set of places (a remote: token). +CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid); -- The roles the next prep maps. Partial: after a prep there are none. CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0; """ @@ -629,6 +645,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''), # A Python step: it adds the column only where it is missing. (26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is " "fetched", lambda conn: _add_posting_url(conn)), + # A Python step: it adds the columns only where they are missing, and seeds + # Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3). + (27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere " + "place: where a remote hire may live", lambda conn: _add_remote_area(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -878,6 +898,40 @@ def _add_posting_url(conn: sqlite3.Connection) -> None: conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''") +def _add_remote_area(conn: sqlite3.Connection) -> None: + """Migration 27's step: `roles.remote_model` empty and + `roles.remote_where_model` NULL for every role, so `extract` reads the + roles it has read once more for the area; `role_remote`, empty until + the next prep; and Anywhere, seeded with the rest of the gazetteer. + + Prep now writes `remote` outright from the model, the host's word and + the place line, where it used to keep a filled column. A board role's + `remote` without a host word becomes its `remote_host`, which a pull + keeps until the host states another word: on live 1,280 of the 3,880 + open ones are given by nothing prep reads now, mostly Greenhouse, Oracle + and Workday roles an older reader mapped. A search role's `remote_host` + its reader no longer sends is an older reading of its title ("... + Remote Sensing ...", 41 roles), and is cleared for prep to read the + title again. Runs twice without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} + if "remote_model" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''") + if "remote_where_model" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT") + kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' " + "AND remote != '' AND source != 'search'").rowcount + cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' " + "AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL" + ).rowcount + log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d", + kept, cleared) + conn.execute("""CREATE TABLE IF NOT EXISTS role_remote ( + uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE, + place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE, + PRIMARY KEY (uid, place_id))""") + gazetteer.seed(conn) + + def _add_all_roles(conn: sqlite3.Connection) -> None: """Migration 21's step. The column; the tag for every company that is a favourite of any agent, since favourite no longer widens what a pull diff --git a/app/storage/store.py b/app/storage/store.py index 9ecd17d..c0d0596 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max") #: What `extract` may fill. Only an empty column is written: a board's own #: structured field wins over what a model read out of the prose, and the next -#: upsert from that board would put it back anyway. `function` and -#: `seniority` are not here either, for the opposite reason — see +#: upsert from that board would put it back anyway. `function`, +#: `seniority` and `remote` are not here, for the opposite reason — see #: `set_extracted`. -_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") +_EXTRACT_TEXT = ("place", "country", "currency", "permit") #: How many replies may leave a role unanswered before `extract` stops #: offering it. newsdesk's number, for newsdesk's reason: a row the model @@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit") GIVE_UP_AFTER = 3 #: A role `extract` has yet to read: still open, no copy of another -#: (`rebuild_dupes`), not given up, and either never answered or without a -#: model level. The second half makes `extract` the afterburner for the level: -#: a role read before the level was asked for is read again, and only fills -#: what is still empty (`set_extracted`). A reply that names no level leaves -#: the role waiting and is charged a try (`extract.apply_batch`), so it ends. -_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL " - f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}") +#: (`rebuild_dupes`), not given up, and either never answered, without a +#: model level, or never asked where a remote hire may live +#: (`remote_where_model` NULL). The last two make `extract` the afterburner +#: for the level and the area: a role read before they were asked for is +#: read again, and only fills what is still empty (`set_extracted`). A reply +#: that names no level leaves the role waiting and is charged a try +#: (`extract.apply_batch`), so it ends. +_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) " + f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}") #: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged. VOTES = (-1, 0, 1, None) @@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray: return np.frombuffer(blob, dtype=_DTYPE) -#: A role with no place the gazetteer resolved, over `roles r`. -_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)" +def _unplaced(gaz: places.Gazetteer) -> str: + """A role with no place the gazetteer resolved, over `roles r`. Anywhere + is no place for this, should a person ever alias it.""" + other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else "" + return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})" + + +#: A remote role whose area is stated (`role_remote`), over `roles r`. +_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))" + + +def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]: + """A remote role whose area holds one of these places, over `table`.""" + return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote " + "WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))]) def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: @@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: filter does (`_places_sql`): a country gate is met by a role whose `country` is one of its codes or that has a place in one of them, so `DK` is met by `Copenhagen; London`. Its column is empty only when the - role has neither. A place gate's value is a place token, bare text a - `place:`.""" + role has neither. A `require` gate on country reads a remote role whose + area is stated through the area alone, as the remote chips do + (`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)" + meet Europe, "Remote US" does not, and neither is empty. A place gate's + value is a place token, bare text a `place:`. + + `remote_from` takes place tokens without their `remote:` (`country:DK`, + `anywhere`) and matches as the remote chips do. It is empty for a role + whose remote is unknown, and for a remote role with no area stated; a + hybrid or onsite role fails it.""" column = f"r.{gate.field}" values = [str(v).strip().lower() for v in gate.values if str(v).strip()] marks = ", ".join("?" for _ in values) or "NULL" if gate.field == "salary_min": floor = min(_int_or_none(v) or 0 for v in gate.values) return f"{column} IS NULL", f"{column} >= ?", [floor] + if gate.field == "remote_from": + tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()] + match, params = _places_sql(tokens, gaz, table="r") + return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params if gate.field == "country": - match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r", - codes=[v.upper() for v in values]) - return f"(r.country = '' AND {_UNPLACED})", match, params + tokens = [f"country:{v}" for v in values] + match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values]) + empty = f"(r.country = '' AND {_unplaced(gaz)})" + if gate.mode != "require": + return empty, match, params + named = {p for t in tokens for p in gaz.lookup(t) or ()} + area, more = _areas_sql(gaz.overlap(named) if named else (), "r") + return (f"({empty} AND NOT {_HAS_AREA})", + f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params]) if gate.field == "place": match, params = _places_sql(values, gaz, table="r") return "r.place = ''", match, params @@ -231,7 +264,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]: def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles", - codes: Sequence[str] = ()) -> tuple[str, list[Any]]: + codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]: """Any of the place tokens over `table`, as a join over the gazetteer: a role with a place under one of the places the tokens name (`country:DK` every place in Denmark, `region:skane` every town in Skåne, `place:malmo` @@ -239,12 +272,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r role whose board gave a country and no place). `codes` are more values of `country` to accept. A `place:` token the gazetteer does not know is matched on whole words of the role's folded line (`places.match_sql`), - so a town nobody has entered yet is still found.""" + so a town nobody has entered yet is still found. + + A `remote:` token is a remote role whose area overlaps its places + (`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere. + `remote:any` is every remote role that meets or leaves empty each of + `scope`, the viewing agent's `require` gates on country; with none, + every remote role.""" ids: set[int] = set() + areas: set[int] = set() found = {c for c in codes if c} clauses: list[str] = [] params: list[Any] = [] + remote: list[tuple[str, list[Any]]] = [] for token in tokens: + tok = places.parse(token) + if tok.kind == "remote": + if tok.value == "any": + remote.append(_in_scope(scope, gaz, table)) + elif named_area := gaz.lookup(token): + areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area) + continue named = gaz.lookup(token) if named is None: sql, values = places.match_sql(token, place=f"{table}.place_fold", @@ -263,9 +311,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r clauses.insert(1 if ids else 0, f"{table}.country IN (SELECT value FROM json_each(?))") params.insert(1 if ids else 0, json.dumps(sorted(found))) + if areas: + remote.append(_areas_sql(areas, table)) + for sql, values in remote: + clauses.append(sql) + params += values return f"({' OR '.join(clauses) or '0'})", params +def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]: + """`remote:any` over `table`: a remote role that meets or leaves empty + each gate of `scope` (`_gate_sql`), so a remote role with no area and no + place is in every scope.""" + if not scope: + return f"{table}.remote = 'remote'", [] + parts: list[str] = [] + params: list[Any] = [] + for gate in scope: + empty, match, values = _gate_sql(gate, gaz) + parts.append(f"({empty} OR {match})") + params += values + return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' " + f"AND {' AND '.join(parts)})", params) + + def _known_places(tokens: Sequence[str]) -> None: """ValueError for a token that names nothing, which would otherwise match no role without saying why.""" @@ -797,6 +866,7 @@ class Store: role.tags = self._tags_of(uid) role.copies = self._copies_of(uid) role.places = self.role_places(uid) + role.remote_where = self._remote_where([uid]).get(uid, "") return role def _copies_of(self, uid: str) -> list[RoleCopy]: @@ -821,6 +891,25 @@ class Store: f"AND {_NOT_A_PLACE.format(kept='dup_of')} " "GROUP BY dup_of", (json.dumps(list(uids)),))} + def _remote_where(self, uids: Sequence[str]) -> dict[str, str]: + """`uid -> the area as a label` ("Europe", "United States, Canada", + "anywhere", `Gazetteer.area_label`) for the remote roles of `uids` + with an area stated; a role without one is absent. See + `Role.remote_where`.""" + if not uids: + return {} + found: dict[str, list[int]] = {} + for r in self._rows( + """SELECT rr.uid, rr.place_id FROM role_remote rr + JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote' + WHERE rr.uid IN (SELECT value FROM json_each(?)) + ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)): + found.setdefault(str(r["uid"]), []).append(int(r["place_id"])) + if not found: + return {} + gaz = self.gazetteer() + return {uid: gaz.area_label(ids) for uid, ids in found.items()} + def _place_lines(self, uids: Sequence[str]) -> dict[str, str]: """`uid -> the line the feed shows` for a page of roles that have more than one place (`dupes.place_line`); a role with one place or @@ -1025,8 +1114,10 @@ class Store: found = [ScoredRole.from_row(r) for r in self._rows( f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])] by_uid = self._tags_by_uid([r.uid for r in found]) + areas = self._remote_where([r.uid for r in found]) for r in found: r.tags = by_uid.get(r.uid, []) + r.remote_where = areas.get(r.uid, "") return found def roles_page(self, limit: int = 100, company: str | None = None, @@ -1082,10 +1173,12 @@ class Store: by_uid = self._tags_by_uid([f.uid for f in found]) copies = self._copy_counts([f.uid for f in found]) lines = self._place_lines([f.uid for f in found]) + areas = self._remote_where([f.uid for f in found]) for f in found: f.tags = by_uid.get(f.uid, []) f.copies = copies.get(f.uid, 0) f.place = lines.get(f.uid, f.place) + f.remote_where = areas.get(f.uid, "") return found def count_roles(self, company: str | None = None, place: str | None = None, @@ -1219,7 +1312,7 @@ class Store: raise ValueError(f"tag_mode is any or all, not {tag_mode!r}") if places: _known_places(places) - clause, values = _places_sql(places, self.gazetteer()) + clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent)) where.append(clause) params += values sql = f"FROM roles {_WITH_VOTES}" @@ -1233,6 +1326,13 @@ class Store: sql += " WHERE " + " AND ".join(where) return sql, [*joins, *params] + def _scope(self, agent_id: int | None) -> list[Gate]: + """The agent's `require` gates on country: what `remote:any` keeps a + remote role within. None without an agent.""" + found = self.agent(agent_id) if agent_id is not None else None + return [g for g in found.gates if g.field == "country" and g.mode == "require"] \ + if found is not None else [] + def tag_counts(self, agent_id: int | None = None, show_muted: bool = False) -> list[tuple[str, int, int]]: """`(tag, open_count, unread_count)` over open roles, most-used tag @@ -1298,17 +1398,27 @@ class Store: # ----------------------------------------------------------------- prep - def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]: + def prep_rows(self, scope: Literal["due", "open", "all"] = "due", + uids: Sequence[str] | None = None) -> list[sqlite3.Row]: """What `app.prep` maps a role from, and what it holds now: the roles - due (`prep_due` above 0), every open role, or every role.""" + due (`prep_due` above 0), every open role, or every role; of `uids` + only, when given. `remote_area` is the role's `role_remote` as a JSON + list, in order.""" where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope] + params: list[Any] = [] + if uids is not None: + where += " AND uid IN (SELECT value FROM json_each(?))" + params.append(json.dumps(list(uids))) return self._rows( f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place, seniority_host, employment_host, remote_host, country_host, seniority_model, seniority_stated, seniority_title, seniority, - employment, remote, country, - json_extract(raw, '$.places') AS raw_places - FROM roles WHERE {where}""") + employment, remote, country, remote_model, remote_where_model, + json_extract(raw, '$.places') AS raw_places, + (SELECT json_group_array(place_id) FROM ( + SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid + ORDER BY rr.rowid)) AS remote_area + FROM roles WHERE {where}""", params) def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]: """Resolve these roles' own places against the gazetteer, writing the @@ -1329,26 +1439,31 @@ class Store: def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int: """Write what prep mapped, one dict per role with `uid`, `due` (the `prep_due` it read), `company_key`, `seniority_stated`, - `seniority_title`, `employment`, `remote`, `remote_places` and + `seniority_title`, `employment`, `remote`, `remote_area` and `country`. `seniority` is the model's level, else the stated, else - the title's. An empty `remote` or `country` leaves the column as it - is, since `extract` fills an empty one. `remote_places`, what the - place texts say, only fills an empty column: it is a guess from a - line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull - changed the role since it was read. Returns the rows written.""" + the title's. `remote` is written as prep composed it (the model's, + else the host's word, else the place line) and `remote_area`, place + ids, replaces the role's `role_remote`. An empty `country` leaves the + column as it is, since `extract` fills an empty one. `prep_due` goes + back to 0 only when no pull changed the role since it was read. + Returns the rows written.""" with self.transaction() as conn: - return conn.executemany( + written = conn.executemany( """UPDATE roles SET company_key = :company_key, seniority_stated = :seniority_stated, seniority_title = :seniority_title, seniority = COALESCE(NULLIF(seniority_model, ''), NULLIF(:seniority_stated, ''), :seniority_title), employment = :employment, - remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''), - :remote_places), + remote = :remote, country = COALESCE(NULLIF(:country, ''), country), prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END WHERE uid = :uid""", mapped).rowcount + conn.executemany("DELETE FROM role_remote WHERE uid = ?", + [(m["uid"],) for m in mapped]) + conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)", + [(m["uid"], p) for m in mapped for p in m.get("remote_area", ())]) + return written def host_words(self) -> list[tuple[str, str, str, int]]: """`(field, host, word, open roles)` for every host word an open role @@ -1418,7 +1533,8 @@ class Store: where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at") .replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries") .replace("dup_of", "r.dup_of") - .replace("seniority_model", "r.seniority_model")] + .replace("seniority_model", "r.seniority_model") + .replace("remote_where_model", "r.remote_where_model")] if uids is not None: where.append("r.uid IN (SELECT value FROM json_each(?))") return " AND ".join(where), [json.dumps(list(uids))] @@ -1468,7 +1584,11 @@ class Store: the rule that then keeps a later pull from putting the guess back. `seniority` is the other: a non-empty answer is `seniority_model`, which comes before the stated and the title's level, so it becomes - `seniority` too.""" + `seniority` too. `remote` and `remote_where` are written outright as + `remote_model` and `remote_where_model` (a JSON list, `[]` for none), + even when empty, since an answer is what the model read this time; + `remote` and the area are composed from them by prep, so the role is + marked due (`prep.run_roles`, which `extract` calls).""" with self.transaction() as conn: row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone() if row is None: @@ -1491,7 +1611,11 @@ class Store: languages = [str(code) for code in fields.get("languages") or []] if languages and json.loads(row["languages"] or "[]") == []: sets["languages"] = json.dumps(languages) - filled = len(sets) - ("seniority" in sets) + remote = str(fields.get("remote") or "") + where = [str(name) for name in fields.get("remote_where") or []] + filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where) + sets["remote_model"] = remote + sets["remote_where_model"] = json.dumps(where) if "place" in sets: sets["place_key"] = place_key(sets["place"]) sets["place_fold"] = places.fold(sets["place"]) @@ -1499,8 +1623,8 @@ class Store: gazetteer.texts({"place": sets["place"]})) sets["extracted_at"] = self.now() conn.execute( - f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?", - (*sets.values(), uid)) + f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, " + "prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid)) return filled def extract_missed(self, uids: Sequence[str], why: str = "") -> int: @@ -2129,6 +2253,18 @@ class Store: "WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' " "GROUP BY place, country")] + def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]: + """`(area, open roles)` for every remote area an open remote role + holds, the area as place ids in order: what `places.suggest` counts + the remote tokens from. Copies are not counted, as in the feed.""" + found: dict[str, list[int]] = {} + for r in self._rows( + "SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid " + "WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' " + "ORDER BY rr.uid, rr.rowid"): + found.setdefault(str(r[0]), []).append(int(r[1])) + return list(Counter(tuple(ids) for ids in found.values()).items()) + def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: """Per subscription id, its open roles counted by `country`, copies included: what a board or a search holds, not what the feed lists.""" diff --git a/tests/test_companies.py b/tests/test_companies.py index 979fb94..6d9993c 100644 --- a/tests/test_companies.py +++ b/tests/test_companies.py @@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> "VALUES (?, ?, '', '')", (key, name)) conn.execute("PRAGMA user_version = 19") reopened = Store(path) - assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26] + assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27] assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(reopened.companies()) == 3 @@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp conn.execute("ALTER TABLE companies DROP COLUMN all_countries") conn.execute("PRAGMA user_version = 20") reopened = Store(path) - assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26] + assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27] # The tag of 21 is All countries and All roles since 24. tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0} assert {c.key: c.all_roles for c in reopened.companies()} == tagged diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index c462cf6..6cb7251 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index 8b4836e..12c5de1 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None: conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms") conn.execute("PRAGMA user_version = 22") migrated = Store(path) - assert migrated.migrations_applied == [23, 24, 25, 26] + assert migrated.migrations_applied == [23, 24, 25, 26, 27] assert migrated.agents()[0].fetch_terms == [] diff --git a/tests/test_overrides.py b/tests/test_overrides.py index e2b75e9..4544530 100644 --- a/tests/test_overrides.py +++ b/tests/test_overrides.py @@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'") conn.execute("PRAGMA user_version = 23") migrated = Store(path) - assert migrated.migrations_applied == [24, 25, 26] + assert migrated.migrations_applied == [24, 25, 26, 27] flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()} assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)} params = {s.target: s.params for s in migrated.subscriptions(source="ats")} diff --git a/tests/test_storage.py b/tests/test_storage.py index a3b3292..d1f9305 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 26 + assert LATEST == 27 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> migrated = Store(path) - assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26] + assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27] assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] assert migrated.role_places(stated.uid) == [] @@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level( migrated = Store(path) - assert migrated.migrations_applied == [25, 26] + assert migrated.migrations_applied == [25, 26, 27] with sqlite3.connect(path) as conn: found = {sid: rest for sid, *rest in conn.execute( "SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority " @@ -748,12 +748,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path) migrated = Store(path) - assert migrated.migrations_applied == [26] + assert migrated.migrations_applied == [26, 27] role = migrated.role(posting("a").uid) assert role is not None and role.posting_url == "" with sqlite3.connect(path) as conn: conn.execute("PRAGMA user_version = 25") # the column is there already - assert Store(path).migrations_applied == [26] + assert Store(path).migrations_applied == [26, 27] def test_a_pull_keeps_the_posting_url(store: Store) -> None: