Add migration 27 and read a remote role's area in the chips, the gates and the rows

roles.remote_model, roles.remote_where_model and role_remote, with a stored remote no host word gives kept as the host's word. The remote tokens filter over role_remote, remote_from and a require gate on country read a remote role through its area, extract waits while remote_where_model is NULL and writes the model's remote outright, and rows carry remote_where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 16:00:06 +02:00
1 parent 189cf81350
commit 57cac5f896
8 files changed
+262 -62

No files matched your search

+175 -39
View File
@@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway. `function` and
#: `seniority` are not here either, for the opposite reason — see
#: upsert from that board would put it back anyway. `function`,
#: `seniority` and `remote` are not here, for the opposite reason — see
#: `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
@@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered or without a
#: model level. The second half makes `extract` the afterburner for the level:
#: a role read before the level was asked for is read again, and only fills
#: what is still empty (`set_extracted`). A reply that names no level leaves
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: (`rebuild_dupes`), not given up, and either never answered, without a
#: model level, or never asked where a remote hire may live
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
#: for the level and the area: a role read before they were asked for is
#: read again, and only fills what is still empty (`set_extracted`). A reply
#: that names no level leaves the role waiting and is charged a try
#: (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
@@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _unplaced(gaz: places.Gazetteer) -> str:
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
is no place for this, should a person ever alias it."""
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
#: A remote role whose area is stated (`role_remote`), over `roles r`.
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
"""A remote role whose area holds one of these places, over `table`."""
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
@@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
role has neither. A `require` gate on country reads a remote role whose
area is stated through the area alone, as the remote chips do
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
meet Europe, "Remote US" does not, and neither is empty. A place gate's
value is a place token, bare text a `place:`.
`remote_from` takes place tokens without their `remote:` (`country:DK`,
`anywhere`) and matches as the remote chips do. It is empty for a role
whose remote is unknown, and for a remote role with no area stated; a
hybrid or onsite role fails it."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "remote_from":
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
match, params = _places_sql(tokens, gaz, table="r")
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
tokens = [f"country:{v}" for v in values]
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
empty = f"(r.country = '' AND {_unplaced(gaz)})"
if gate.mode != "require":
return empty, match, params
named = {p for t in tokens for p in gaz.lookup(t) or ()}
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
return (f"({empty} AND NOT {_HAS_AREA})",
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
if gate.field == "place":
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
@@ -231,7 +264,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
@@ -239,12 +272,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
so a town nobody has entered yet is still found.
A `remote:` token is a remote role whose area overlaps its places
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
`remote:any` is every remote role that meets or leaves empty each of
`scope`, the viewing agent's `require` gates on country; with none,
every remote role."""
ids: set[int] = set()
areas: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
remote: list[tuple[str, list[Any]]] = []
for token in tokens:
tok = places.parse(token)
if tok.kind == "remote":
if tok.value == "any":
remote.append(_in_scope(scope, gaz, table))
elif named_area := gaz.lookup(token):
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
continue
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
@@ -263,9 +311,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
if areas:
remote.append(_areas_sql(areas, table))
for sql, values in remote:
clauses.append(sql)
params += values
return f"({' OR '.join(clauses) or '0'})", params
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
"""`remote:any` over `table`: a remote role that meets or leaves empty
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
place is in every scope."""
if not scope:
return f"{table}.remote = 'remote'", []
parts: list[str] = []
params: list[Any] = []
for gate in scope:
empty, match, values = _gate_sql(gate, gaz)
parts.append(f"({empty} OR {match})")
params += values
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
f"AND {' AND '.join(parts)})", params)
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
@@ -797,6 +866,7 @@ class Store:
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
role.remote_where = self._remote_where([uid]).get(uid, "")
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -821,6 +891,25 @@ class Store:
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
"GROUP BY dup_of", (json.dumps(list(uids)),))}
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the area as a label` ("Europe", "United States, Canada",
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
with an area stated; a role without one is absent. See
`Role.remote_where`."""
if not uids:
return {}
found: dict[str, list[int]] = {}
for r in self._rows(
"""SELECT rr.uid, rr.place_id FROM role_remote rr
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
WHERE rr.uid IN (SELECT value FROM json_each(?))
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
if not found:
return {}
gaz = self.gazetteer()
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the line the feed shows` for a page of roles that have
more than one place (`dupes.place_line`); a role with one place or
@@ -1025,8 +1114,10 @@ class Store:
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
areas = self._remote_where([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
r.remote_where = areas.get(r.uid, "")
return found
def roles_page(self, limit: int = 100, company: str | None = None,
@@ -1082,10 +1173,12 @@ class Store:
by_uid = self._tags_by_uid([f.uid for f in found])
copies = self._copy_counts([f.uid for f in found])
lines = self._place_lines([f.uid for f in found])
areas = self._remote_where([f.uid for f in found])
for f in found:
f.tags = by_uid.get(f.uid, [])
f.copies = copies.get(f.uid, 0)
f.place = lines.get(f.uid, f.place)
f.remote_where = areas.get(f.uid, "")
return found
def count_roles(self, company: str | None = None, place: str | None = None,
@@ -1219,7 +1312,7 @@ class Store:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1233,6 +1326,13 @@ class Store:
sql += " WHERE " + " AND ".join(where)
return sql, [*joins, *params]
def _scope(self, agent_id: int | None) -> list[Gate]:
"""The agent's `require` gates on country: what `remote:any` keeps a
remote role within. None without an agent."""
found = self.agent(agent_id) if agent_id is not None else None
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
if found is not None else []
def tag_counts(self, agent_id: int | None = None,
show_muted: bool = False) -> list[tuple[str, int, int]]:
"""`(tag, open_count, unread_count)` over open roles, most-used tag
@@ -1298,17 +1398,27 @@ class Store:
# ----------------------------------------------------------------- prep
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
"""What `app.prep` maps a role from, and what it holds now: the roles
due (`prep_due` above 0), every open role, or every role."""
due (`prep_due` above 0), every open role, or every role; of `uids`
only, when given. `remote_area` is the role's `role_remote` as a JSON
list, in order."""
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
params: list[Any] = []
if uids is not None:
where += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
return self._rows(
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country,
json_extract(raw, '$.places') AS raw_places
FROM roles WHERE {where}""")
employment, remote, country, remote_model, remote_where_model,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
ORDER BY rr.rowid)) AS remote_area
FROM roles WHERE {where}""", params)
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve these roles' own places against the gazetteer, writing the
@@ -1329,26 +1439,31 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_places` and
`seniority_title`, `employment`, `remote`, `remote_area` and
`country`. `seniority` is the model's level, else the stated, else
the title's. An empty `remote` or `country` leaves the column as it
is, since `extract` fills an empty one. `remote_places`, what the
place texts say, only fills an empty column: it is a guess from a
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
changed the role since it was read. Returns the rows written."""
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
column as it is, since `extract` fills an empty one. `prep_due` goes
back to 0 only when no pull changed the role since it was read.
Returns the rows written."""
with self.transaction() as conn:
return conn.executemany(
written = conn.executemany(
"""UPDATE roles SET company_key = :company_key,
seniority_stated = :seniority_stated,
seniority_title = :seniority_title,
seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
:remote_places),
remote = :remote,
country = COALESCE(NULLIF(:country, ''), country),
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
[(m["uid"],) for m in mapped])
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
return written
def host_words(self) -> list[tuple[str, str, str, int]]:
"""`(field, host, word, open roles)` for every host word an open role
@@ -1418,7 +1533,8 @@ class Store:
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")]
.replace("seniority_model", "r.seniority_model")
.replace("remote_where_model", "r.remote_where_model")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1468,7 +1584,11 @@ class Store:
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
`seniority` too. `remote` and `remote_where` are written outright as
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
even when empty, since an answer is what the model read this time;
`remote` and the area are composed from them by prep, so the role is
marked due (`prep.run_roles`, which `extract` calls)."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1491,7 +1611,11 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets) - ("seniority" in sets)
remote = str(fields.get("remote") or "")
where = [str(name) for name in fields.get("remote_where") or []]
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
sets["remote_model"] = remote
sets["remote_where_model"] = json.dumps(where)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
@@ -1499,8 +1623,8 @@ class Store:
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
@@ -2129,6 +2253,18 @@ class Store:
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
"GROUP BY place, country")]
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
"""`(area, open roles)` for every remote area an open remote role
holds, the area as place ids in order: what `places.suggest` counts
the remote tokens from. Copies are not counted, as in the feed."""
found: dict[str, list[int]] = {}
for r in self._rows(
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
"ORDER BY rr.uid, rr.rowid"):
found.setdefault(str(r[0]), []).append(int(r[1]))
return list(Counter(tuple(ids) for ids in found.values()).items())
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`, copies
included: what a board or a search holds, not what the feed lists."""