Add migration 27 and read a remote role's area in the chips, the gates and the rows

roles.remote_model, roles.remote_where_model and role_remote, with a stored remote no host word gives kept as the host's word. The remote tokens filter over role_remote, remote_from and a require gate on country read a remote role through its area, extract waits while remote_where_model is NULL and writes the model's remote outright, and rows carry remote_where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 16:00:06 +02:00
1 parent 189cf81350
commit 57cac5f896
8 files changed
+262 -62

No files matched your search

+15 -5
View File
@@ -970,6 +970,10 @@ class Role(Model):
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
#: Where a remote hire may live, as a label: "anywhere", "Europe",
#: "United States, Canada". '' when the role is not remote or its area is
#: not stated. Not a column: filled by `Store` from `role_remote`.
remote_where: str = ""
@computed_field # type: ignore[prop-decorator]
@property
@@ -1076,6 +1080,9 @@ class RoleRow(Model):
excerpt: str = ""
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
copies: int = 0
#: As `Role.remote_where`: the area a remote hire may live in, '' when
#: the role is not remote or does not say.
remote_where: str = ""
@computed_field # type: ignore[prop-decorator]
@property
@@ -1167,11 +1174,14 @@ class Application(Model):
#: The role columns a gate can name. `company` matches on the company key;
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"]
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words;
#: `remote_from` takes place tokens (`country:DK`, `country:EU`, `anywhere`)
#: and is met by a remote role whose area overlaps one, as a `remote:` chip
#: is; the rest are case-insensitive equality, except `salary_min` (at least
#: the value) and `languages` (every language the role requires is one of
#: the values).
GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function",
"salary_min", "currency", "permit", "languages", "company"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer"]
+58 -4
View File
@@ -118,7 +118,9 @@ CREATE TABLE IF NOT EXISTS roles (
remote_host TEXT NOT NULL DEFAULT '',
country_host TEXT NOT NULL DEFAULT '',
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0
prep_due INTEGER NOT NULL DEFAULT 0,
remote_model TEXT NOT NULL DEFAULT '',
remote_where_model TEXT
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
@@ -129,9 +131,12 @@ CREATE TABLE IF NOT EXISTS roles (
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
-- extract's remote, which wins over the host's word and the place line, and
-- remote_where_model the names it read for where a remote hire may live (a
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
-- columns: SQLite cannot drop a column a comment precedes, which the
-- migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
@@ -177,6 +182,15 @@ CREATE TABLE IF NOT EXISTS role_places (
url TEXT NOT NULL DEFAULT ''
);
-- A remote role's area, the places a remote hire may live in (app/prep/area.py),
-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with
-- no row is not remote, or remote with its area not stated.
CREATE TABLE IF NOT EXISTS role_remote (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (uid, place_id)
);
-- The open vocabulary summarise reads off a posting: the role family, the
-- technologies and skills it names, and the domain. What the feed's filter
-- row is built from; see Store.set_tags and Store.tag_counts.
@@ -424,6 +438,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
-- The remote roles whose area holds a set of places (a remote: token).
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
"""
@@ -629,6 +645,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# A Python step: it adds the column only where it is missing.
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
"fetched", lambda conn: _add_posting_url(conn)),
# A Python step: it adds the columns only where they are missing, and seeds
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -878,6 +898,40 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
def _add_remote_area(conn: sqlite3.Connection) -> None:
"""Migration 27's step: `roles.remote_model` empty and
`roles.remote_where_model` NULL for every role, so `extract` reads the
roles it has read once more for the area; `role_remote`, empty until
the next prep; and Anywhere, seeded with the rest of the gazetteer.
Prep now writes `remote` outright from the model, the host's word and
the place line, where it used to keep a filled column. A board role's
`remote` without a host word becomes its `remote_host`, which a pull
keeps until the host states another word: on live 1,280 of the 3,880
open ones are given by nothing prep reads now, mostly Greenhouse, Oracle
and Workday roles an older reader mapped. A search role's `remote_host`
its reader no longer sends is an older reading of its title ("...
Remote Sensing ...", 41 roles), and is cleared for prep to read the
title again. Runs twice without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
if "remote_model" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''")
if "remote_where_model" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT")
kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' "
"AND remote != '' AND source != 'search'").rowcount
cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' "
"AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL"
).rowcount
log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d",
kept, cleared)
conn.execute("""CREATE TABLE IF NOT EXISTS role_remote (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (uid, place_id))""")
gazetteer.seed(conn)
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+175 -39
View File
@@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway. `function` and
#: `seniority` are not here either, for the opposite reason — see
#: upsert from that board would put it back anyway. `function`,
#: `seniority` and `remote` are not here, for the opposite reason — see
#: `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
@@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered or without a
#: model level. The second half makes `extract` the afterburner for the level:
#: a role read before the level was asked for is read again, and only fills
#: what is still empty (`set_extracted`). A reply that names no level leaves
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: (`rebuild_dupes`), not given up, and either never answered, without a
#: model level, or never asked where a remote hire may live
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
#: for the level and the area: a role read before they were asked for is
#: read again, and only fills what is still empty (`set_extracted`). A reply
#: that names no level leaves the role waiting and is charged a try
#: (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
@@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _unplaced(gaz: places.Gazetteer) -> str:
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
is no place for this, should a person ever alias it."""
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
#: A remote role whose area is stated (`role_remote`), over `roles r`.
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
"""A remote role whose area holds one of these places, over `table`."""
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
@@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
role has neither. A `require` gate on country reads a remote role whose
area is stated through the area alone, as the remote chips do
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
meet Europe, "Remote US" does not, and neither is empty. A place gate's
value is a place token, bare text a `place:`.
`remote_from` takes place tokens without their `remote:` (`country:DK`,
`anywhere`) and matches as the remote chips do. It is empty for a role
whose remote is unknown, and for a remote role with no area stated; a
hybrid or onsite role fails it."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "remote_from":
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
match, params = _places_sql(tokens, gaz, table="r")
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
tokens = [f"country:{v}" for v in values]
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
empty = f"(r.country = '' AND {_unplaced(gaz)})"
if gate.mode != "require":
return empty, match, params
named = {p for t in tokens for p in gaz.lookup(t) or ()}
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
return (f"({empty} AND NOT {_HAS_AREA})",
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
if gate.field == "place":
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
@@ -231,7 +264,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
@@ -239,12 +272,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
so a town nobody has entered yet is still found.
A `remote:` token is a remote role whose area overlaps its places
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
`remote:any` is every remote role that meets or leaves empty each of
`scope`, the viewing agent's `require` gates on country; with none,
every remote role."""
ids: set[int] = set()
areas: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
remote: list[tuple[str, list[Any]]] = []
for token in tokens:
tok = places.parse(token)
if tok.kind == "remote":
if tok.value == "any":
remote.append(_in_scope(scope, gaz, table))
elif named_area := gaz.lookup(token):
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
continue
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
@@ -263,9 +311,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
if areas:
remote.append(_areas_sql(areas, table))
for sql, values in remote:
clauses.append(sql)
params += values
return f"({' OR '.join(clauses) or '0'})", params
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
"""`remote:any` over `table`: a remote role that meets or leaves empty
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
place is in every scope."""
if not scope:
return f"{table}.remote = 'remote'", []
parts: list[str] = []
params: list[Any] = []
for gate in scope:
empty, match, values = _gate_sql(gate, gaz)
parts.append(f"({empty} OR {match})")
params += values
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
f"AND {' AND '.join(parts)})", params)
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
@@ -797,6 +866,7 @@ class Store:
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
role.remote_where = self._remote_where([uid]).get(uid, "")
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -821,6 +891,25 @@ class Store:
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
"GROUP BY dup_of", (json.dumps(list(uids)),))}
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the area as a label` ("Europe", "United States, Canada",
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
with an area stated; a role without one is absent. See
`Role.remote_where`."""
if not uids:
return {}
found: dict[str, list[int]] = {}
for r in self._rows(
"""SELECT rr.uid, rr.place_id FROM role_remote rr
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
WHERE rr.uid IN (SELECT value FROM json_each(?))
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
if not found:
return {}
gaz = self.gazetteer()
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the line the feed shows` for a page of roles that have
more than one place (`dupes.place_line`); a role with one place or
@@ -1025,8 +1114,10 @@ class Store:
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
areas = self._remote_where([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
r.remote_where = areas.get(r.uid, "")
return found
def roles_page(self, limit: int = 100, company: str | None = None,
@@ -1082,10 +1173,12 @@ class Store:
by_uid = self._tags_by_uid([f.uid for f in found])
copies = self._copy_counts([f.uid for f in found])
lines = self._place_lines([f.uid for f in found])
areas = self._remote_where([f.uid for f in found])
for f in found:
f.tags = by_uid.get(f.uid, [])
f.copies = copies.get(f.uid, 0)
f.place = lines.get(f.uid, f.place)
f.remote_where = areas.get(f.uid, "")
return found
def count_roles(self, company: str | None = None, place: str | None = None,
@@ -1219,7 +1312,7 @@ class Store:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1233,6 +1326,13 @@ class Store:
sql += " WHERE " + " AND ".join(where)
return sql, [*joins, *params]
def _scope(self, agent_id: int | None) -> list[Gate]:
"""The agent's `require` gates on country: what `remote:any` keeps a
remote role within. None without an agent."""
found = self.agent(agent_id) if agent_id is not None else None
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
if found is not None else []
def tag_counts(self, agent_id: int | None = None,
show_muted: bool = False) -> list[tuple[str, int, int]]:
"""`(tag, open_count, unread_count)` over open roles, most-used tag
@@ -1298,17 +1398,27 @@ class Store:
# ----------------------------------------------------------------- prep
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
"""What `app.prep` maps a role from, and what it holds now: the roles
due (`prep_due` above 0), every open role, or every role."""
due (`prep_due` above 0), every open role, or every role; of `uids`
only, when given. `remote_area` is the role's `role_remote` as a JSON
list, in order."""
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
params: list[Any] = []
if uids is not None:
where += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
return self._rows(
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country,
json_extract(raw, '$.places') AS raw_places
FROM roles WHERE {where}""")
employment, remote, country, remote_model, remote_where_model,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
ORDER BY rr.rowid)) AS remote_area
FROM roles WHERE {where}""", params)
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve these roles' own places against the gazetteer, writing the
@@ -1329,26 +1439,31 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_places` and
`seniority_title`, `employment`, `remote`, `remote_area` and
`country`. `seniority` is the model's level, else the stated, else
the title's. An empty `remote` or `country` leaves the column as it
is, since `extract` fills an empty one. `remote_places`, what the
place texts say, only fills an empty column: it is a guess from a
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
changed the role since it was read. Returns the rows written."""
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
column as it is, since `extract` fills an empty one. `prep_due` goes
back to 0 only when no pull changed the role since it was read.
Returns the rows written."""
with self.transaction() as conn:
return conn.executemany(
written = conn.executemany(
"""UPDATE roles SET company_key = :company_key,
seniority_stated = :seniority_stated,
seniority_title = :seniority_title,
seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
:remote_places),
remote = :remote,
country = COALESCE(NULLIF(:country, ''), country),
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
[(m["uid"],) for m in mapped])
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
return written
def host_words(self) -> list[tuple[str, str, str, int]]:
"""`(field, host, word, open roles)` for every host word an open role
@@ -1418,7 +1533,8 @@ class Store:
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")]
.replace("seniority_model", "r.seniority_model")
.replace("remote_where_model", "r.remote_where_model")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1468,7 +1584,11 @@ class Store:
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
`seniority` too. `remote` and `remote_where` are written outright as
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
even when empty, since an answer is what the model read this time;
`remote` and the area are composed from them by prep, so the role is
marked due (`prep.run_roles`, which `extract` calls)."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1491,7 +1611,11 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets) - ("seniority" in sets)
remote = str(fields.get("remote") or "")
where = [str(name) for name in fields.get("remote_where") or []]
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
sets["remote_model"] = remote
sets["remote_where_model"] = json.dumps(where)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
@@ -1499,8 +1623,8 @@ class Store:
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
@@ -2129,6 +2253,18 @@ class Store:
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
"GROUP BY place, country")]
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
"""`(area, open roles)` for every remote area an open remote role
holds, the area as place ids in order: what `places.suggest` counts
the remote tokens from. Copies are not counted, as in the feed."""
found: dict[str, list[int]] = {}
for r in self._rows(
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
"ORDER BY rr.uid, rr.rowid"):
found.setdefault(str(r[0]), []).append(int(r[1]))
return list(Counter(tuple(ids) for ids in found.values()).items())
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`, copies
included: what a board or a search holds, not what the feed lists."""
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+1 -1
View File
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+1 -1
View File
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26]
assert migrated.migrations_applied == [23, 24, 25, 26, 27]
assert migrated.agents()[0].fetch_terms == []
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26]
assert migrated.migrations_applied == [24, 25, 26, 27]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+9 -9
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 26
assert LATEST == 27
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26]
assert migrated.migrations_applied == [25, 26, 27]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -748,12 +748,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26]
assert migrated.migrations_applied == [26, 27]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26]
assert Store(path).migrations_applied == [26, 27]
def test_a_pull_keeps_the_posting_url(store: Store) -> None: