Add migration 27 and read a remote role's area in the chips, the gates and the rows
roles.remote_model, roles.remote_where_model and role_remote, with a stored remote no host word gives kept as the host's word. The remote tokens filter over role_remote, remote_from and a require gate on country read a remote role through its area, extract waits while remote_where_model is NULL and writes the model's remote outright, and rows carry remote_where. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
189cf81350
commit
57cac5f896
8 files changed
+262
-62
No files matched your search
+15
-5
@@ -970,6 +970,10 @@ class Role(Model):
|
||||
#: Every place of the role, in the host's order. Not a column: filled by
|
||||
#: `Store.role` from `role_places`.
|
||||
places: list[RolePlace] = []
|
||||
#: Where a remote hire may live, as a label: "anywhere", "Europe",
|
||||
#: "United States, Canada". '' when the role is not remote or its area is
|
||||
#: not stated. Not a column: filled by `Store` from `role_remote`.
|
||||
remote_where: str = ""
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
@@ -1076,6 +1080,9 @@ class RoleRow(Model):
|
||||
excerpt: str = ""
|
||||
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
|
||||
copies: int = 0
|
||||
#: As `Role.remote_where`: the area a remote hire may live in, '' when
|
||||
#: the role is not remote or does not say.
|
||||
remote_where: str = ""
|
||||
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
@@ -1167,11 +1174,14 @@ class Application(Model):
|
||||
|
||||
#: The role columns a gate can name. `company` matches on the company key;
|
||||
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
|
||||
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
|
||||
#: case-insensitive equality, except `salary_min` (at least the value) and
|
||||
#: `languages` (every language the role requires is one of the values).
|
||||
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
||||
"currency", "permit", "languages", "company"]
|
||||
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words;
|
||||
#: `remote_from` takes place tokens (`country:DK`, `country:EU`, `anywhere`)
|
||||
#: and is met by a remote role whose area overlaps one, as a `remote:` chip
|
||||
#: is; the rest are case-insensitive equality, except `salary_min` (at least
|
||||
#: the value) and `languages` (every language the role requires is one of
|
||||
#: the values).
|
||||
GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function",
|
||||
"salary_min", "currency", "permit", "languages", "company"]
|
||||
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
||||
|
||||
GateMode = Literal["require", "any", "prefer"]
|
||||
|
||||
+58
-4
@@ -118,7 +118,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
prep_due INTEGER NOT NULL DEFAULT 0,
|
||||
remote_model TEXT NOT NULL DEFAULT '',
|
||||
remote_where_model TEXT
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
@@ -129,9 +131,12 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
-- three levels behind roles.seniority: the model's (extract), the host's own
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
|
||||
-- among the last columns: SQLite cannot drop a column a comment precedes,
|
||||
-- which the migration tests do.
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
|
||||
-- extract's remote, which wins over the host's word and the place line, and
|
||||
-- remote_where_model the names it read for where a remote hire may live (a
|
||||
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
|
||||
-- columns: SQLite cannot drop a column a comment precedes, which the
|
||||
-- migration tests do.
|
||||
|
||||
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
||||
-- region or a city, in the place parent_id names. key is unique per kind:
|
||||
@@ -177,6 +182,15 @@ CREATE TABLE IF NOT EXISTS role_places (
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- A remote role's area, the places a remote hire may live in (app/prep/area.py),
|
||||
-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with
|
||||
-- no row is not remote, or remote with its area not stated.
|
||||
CREATE TABLE IF NOT EXISTS role_remote (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (uid, place_id)
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
-- technologies and skills it names, and the domain. What the feed's filter
|
||||
-- row is built from; see Store.set_tags and Store.tag_counts.
|
||||
@@ -424,6 +438,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The remote roles whose area holds a set of places (a remote: token).
|
||||
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
@@ -629,6 +645,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
# A Python step: it adds the column only where it is missing.
|
||||
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
|
||||
"fetched", lambda conn: _add_posting_url(conn)),
|
||||
# A Python step: it adds the columns only where they are missing, and seeds
|
||||
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
|
||||
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
|
||||
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -878,6 +898,40 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||
|
||||
|
||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 27's step: `roles.remote_model` empty and
|
||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||
roles it has read once more for the area; `role_remote`, empty until
|
||||
the next prep; and Anywhere, seeded with the rest of the gazetteer.
|
||||
|
||||
Prep now writes `remote` outright from the model, the host's word and
|
||||
the place line, where it used to keep a filled column. A board role's
|
||||
`remote` without a host word becomes its `remote_host`, which a pull
|
||||
keeps until the host states another word: on live 1,280 of the 3,880
|
||||
open ones are given by nothing prep reads now, mostly Greenhouse, Oracle
|
||||
and Workday roles an older reader mapped. A search role's `remote_host`
|
||||
its reader no longer sends is an older reading of its title ("...
|
||||
Remote Sensing ...", 41 roles), and is cleared for prep to read the
|
||||
title again. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
if "remote_model" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''")
|
||||
if "remote_where_model" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT")
|
||||
kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' "
|
||||
"AND remote != '' AND source != 'search'").rowcount
|
||||
cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' "
|
||||
"AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL"
|
||||
).rowcount
|
||||
log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d",
|
||||
kept, cleared)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS role_remote (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (uid, place_id))""")
|
||||
gazetteer.seed(conn)
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
+175
-39
@@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway. `function` and
|
||||
#: `seniority` are not here either, for the opposite reason — see
|
||||
#: upsert from that board would put it back anyway. `function`,
|
||||
#: `seniority` and `remote` are not here, for the opposite reason — see
|
||||
#: `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
@@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: still open, no copy of another
|
||||
#: (`rebuild_dupes`), not given up, and either never answered or without a
|
||||
#: model level. The second half makes `extract` the afterburner for the level:
|
||||
#: a role read before the level was asked for is read again, and only fills
|
||||
#: what is still empty (`set_extracted`). A reply that names no level leaves
|
||||
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
|
||||
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
|
||||
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
#: (`rebuild_dupes`), not given up, and either never answered, without a
|
||||
#: model level, or never asked where a remote hire may live
|
||||
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
|
||||
#: for the level and the area: a role read before they were asked for is
|
||||
#: read again, and only fills what is still empty (`set_extracted`). A reply
|
||||
#: that names no level leaves the role waiting and is charged a try
|
||||
#: (`extract.apply_batch`), so it ends.
|
||||
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
|
||||
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
VOTES = (-1, 0, 1, None)
|
||||
@@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
#: A role with no place the gazetteer resolved, over `roles r`.
|
||||
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
|
||||
def _unplaced(gaz: places.Gazetteer) -> str:
|
||||
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
|
||||
is no place for this, should a person ever alias it."""
|
||||
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
|
||||
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
|
||||
|
||||
|
||||
#: A remote role whose area is stated (`role_remote`), over `roles r`.
|
||||
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
|
||||
|
||||
|
||||
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
|
||||
"""A remote role whose area holds one of these places, over `table`."""
|
||||
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
|
||||
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
@@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
filter does (`_places_sql`): a country gate is met by a role whose
|
||||
`country` is one of its codes or that has a place in one of them, so
|
||||
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
||||
role has neither. A place gate's value is a place token, bare text a
|
||||
`place:`."""
|
||||
role has neither. A `require` gate on country reads a remote role whose
|
||||
area is stated through the area alone, as the remote chips do
|
||||
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
|
||||
meet Europe, "Remote US" does not, and neither is empty. A place gate's
|
||||
value is a place token, bare text a `place:`.
|
||||
|
||||
`remote_from` takes place tokens without their `remote:` (`country:DK`,
|
||||
`anywhere`) and matches as the remote chips do. It is empty for a role
|
||||
whose remote is unknown, and for a remote role with no area stated; a
|
||||
hybrid or onsite role fails it."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "remote_from":
|
||||
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
|
||||
match, params = _places_sql(tokens, gaz, table="r")
|
||||
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
|
||||
if gate.field == "country":
|
||||
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
|
||||
codes=[v.upper() for v in values])
|
||||
return f"(r.country = '' AND {_UNPLACED})", match, params
|
||||
tokens = [f"country:{v}" for v in values]
|
||||
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
|
||||
empty = f"(r.country = '' AND {_unplaced(gaz)})"
|
||||
if gate.mode != "require":
|
||||
return empty, match, params
|
||||
named = {p for t in tokens for p in gaz.lookup(t) or ()}
|
||||
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
|
||||
return (f"({empty} AND NOT {_HAS_AREA})",
|
||||
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
|
||||
if gate.field == "place":
|
||||
match, params = _places_sql(values, gaz, table="r")
|
||||
return "r.place = ''", match, params
|
||||
@@ -231,7 +264,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
|
||||
|
||||
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
||||
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
|
||||
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
|
||||
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
||||
role with a place under one of the places the tokens name (`country:DK`
|
||||
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
||||
@@ -239,12 +272,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
||||
role whose board gave a country and no place). `codes` are more values of
|
||||
`country` to accept. A `place:` token the gazetteer does not know is
|
||||
matched on whole words of the role's folded line (`places.match_sql`),
|
||||
so a town nobody has entered yet is still found."""
|
||||
so a town nobody has entered yet is still found.
|
||||
|
||||
A `remote:` token is a remote role whose area overlaps its places
|
||||
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
|
||||
`remote:any` is every remote role that meets or leaves empty each of
|
||||
`scope`, the viewing agent's `require` gates on country; with none,
|
||||
every remote role."""
|
||||
ids: set[int] = set()
|
||||
areas: set[int] = set()
|
||||
found = {c for c in codes if c}
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
remote: list[tuple[str, list[Any]]] = []
|
||||
for token in tokens:
|
||||
tok = places.parse(token)
|
||||
if tok.kind == "remote":
|
||||
if tok.value == "any":
|
||||
remote.append(_in_scope(scope, gaz, table))
|
||||
elif named_area := gaz.lookup(token):
|
||||
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
|
||||
continue
|
||||
named = gaz.lookup(token)
|
||||
if named is None:
|
||||
sql, values = places.match_sql(token, place=f"{table}.place_fold",
|
||||
@@ -263,9 +311,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
||||
clauses.insert(1 if ids else 0,
|
||||
f"{table}.country IN (SELECT value FROM json_each(?))")
|
||||
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
||||
if areas:
|
||||
remote.append(_areas_sql(areas, table))
|
||||
for sql, values in remote:
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
return f"({' OR '.join(clauses) or '0'})", params
|
||||
|
||||
|
||||
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
|
||||
"""`remote:any` over `table`: a remote role that meets or leaves empty
|
||||
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
|
||||
place is in every scope."""
|
||||
if not scope:
|
||||
return f"{table}.remote = 'remote'", []
|
||||
parts: list[str] = []
|
||||
params: list[Any] = []
|
||||
for gate in scope:
|
||||
empty, match, values = _gate_sql(gate, gaz)
|
||||
parts.append(f"({empty} OR {match})")
|
||||
params += values
|
||||
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
|
||||
f"AND {' AND '.join(parts)})", params)
|
||||
|
||||
|
||||
def _known_places(tokens: Sequence[str]) -> None:
|
||||
"""ValueError for a token that names nothing, which would otherwise match
|
||||
no role without saying why."""
|
||||
@@ -797,6 +866,7 @@ class Store:
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
role.places = self.role_places(uid)
|
||||
role.remote_where = self._remote_where([uid]).get(uid, "")
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
@@ -821,6 +891,25 @@ class Store:
|
||||
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the area as a label` ("Europe", "United States, Canada",
|
||||
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
|
||||
with an area stated; a role without one is absent. See
|
||||
`Role.remote_where`."""
|
||||
if not uids:
|
||||
return {}
|
||||
found: dict[str, list[int]] = {}
|
||||
for r in self._rows(
|
||||
"""SELECT rr.uid, rr.place_id FROM role_remote rr
|
||||
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
|
||||
WHERE rr.uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
|
||||
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
|
||||
if not found:
|
||||
return {}
|
||||
gaz = self.gazetteer()
|
||||
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
|
||||
|
||||
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the line the feed shows` for a page of roles that have
|
||||
more than one place (`dupes.place_line`); a role with one place or
|
||||
@@ -1025,8 +1114,10 @@ class Store:
|
||||
found = [ScoredRole.from_row(r) for r in self._rows(
|
||||
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
areas = self._remote_where([r.uid for r in found])
|
||||
for r in found:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
r.remote_where = areas.get(r.uid, "")
|
||||
return found
|
||||
|
||||
def roles_page(self, limit: int = 100, company: str | None = None,
|
||||
@@ -1082,10 +1173,12 @@ class Store:
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
lines = self._place_lines([f.uid for f in found])
|
||||
areas = self._remote_where([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
f.place = lines.get(f.uid, f.place)
|
||||
f.remote_where = areas.get(f.uid, "")
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
@@ -1219,7 +1312,7 @@ class Store:
|
||||
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
||||
if places:
|
||||
_known_places(places)
|
||||
clause, values = _places_sql(places, self.gazetteer())
|
||||
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
|
||||
where.append(clause)
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES}"
|
||||
@@ -1233,6 +1326,13 @@ class Store:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
return sql, [*joins, *params]
|
||||
|
||||
def _scope(self, agent_id: int | None) -> list[Gate]:
|
||||
"""The agent's `require` gates on country: what `remote:any` keeps a
|
||||
remote role within. None without an agent."""
|
||||
found = self.agent(agent_id) if agent_id is not None else None
|
||||
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
|
||||
if found is not None else []
|
||||
|
||||
def tag_counts(self, agent_id: int | None = None,
|
||||
show_muted: bool = False) -> list[tuple[str, int, int]]:
|
||||
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
||||
@@ -1298,17 +1398,27 @@ class Store:
|
||||
|
||||
# ----------------------------------------------------------------- prep
|
||||
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
|
||||
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
||||
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||
due (`prep_due` above 0), every open role, or every role."""
|
||||
due (`prep_due` above 0), every open role, or every role; of `uids`
|
||||
only, when given. `remote_area` is the role's `role_remote` as a JSON
|
||||
list, in order."""
|
||||
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||
params: list[Any] = []
|
||||
if uids is not None:
|
||||
where += " AND uid IN (SELECT value FROM json_each(?))"
|
||||
params.append(json.dumps(list(uids)))
|
||||
return self._rows(
|
||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country,
|
||||
json_extract(raw, '$.places') AS raw_places
|
||||
FROM roles WHERE {where}""")
|
||||
employment, remote, country, remote_model, remote_where_model,
|
||||
json_extract(raw, '$.places') AS raw_places,
|
||||
(SELECT json_group_array(place_id) FROM (
|
||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||
ORDER BY rr.rowid)) AS remote_area
|
||||
FROM roles WHERE {where}""", params)
|
||||
|
||||
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve these roles' own places against the gazetteer, writing the
|
||||
@@ -1329,26 +1439,31 @@ class Store:
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||
`seniority_title`, `employment`, `remote`, `remote_area` and
|
||||
`country`. `seniority` is the model's level, else the stated, else
|
||||
the title's. An empty `remote` or `country` leaves the column as it
|
||||
is, since `extract` fills an empty one. `remote_places`, what the
|
||||
place texts say, only fills an empty column: it is a guess from a
|
||||
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||
changed the role since it was read. Returns the rows written."""
|
||||
the title's. `remote` is written as prep composed it (the model's,
|
||||
else the host's word, else the place line) and `remote_area`, place
|
||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||
column as it is, since `extract` fills an empty one. `prep_due` goes
|
||||
back to 0 only when no pull changed the role since it was read.
|
||||
Returns the rows written."""
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
written = conn.executemany(
|
||||
"""UPDATE roles SET company_key = :company_key,
|
||||
seniority_stated = :seniority_stated,
|
||||
seniority_title = :seniority_title,
|
||||
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||
:remote_places),
|
||||
remote = :remote,
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
WHERE uid = :uid""", mapped).rowcount
|
||||
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
|
||||
[(m["uid"],) for m in mapped])
|
||||
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
|
||||
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
|
||||
return written
|
||||
|
||||
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||
"""`(field, host, word, open roles)` for every host word an open role
|
||||
@@ -1418,7 +1533,8 @@ class Store:
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")
|
||||
.replace("seniority_model", "r.seniority_model")]
|
||||
.replace("seniority_model", "r.seniority_model")
|
||||
.replace("remote_where_model", "r.remote_where_model")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
@@ -1468,7 +1584,11 @@ class Store:
|
||||
the rule that then keeps a later pull from putting the guess back.
|
||||
`seniority` is the other: a non-empty answer is `seniority_model`,
|
||||
which comes before the stated and the title's level, so it becomes
|
||||
`seniority` too."""
|
||||
`seniority` too. `remote` and `remote_where` are written outright as
|
||||
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
|
||||
even when empty, since an answer is what the model read this time;
|
||||
`remote` and the area are composed from them by prep, so the role is
|
||||
marked due (`prep.run_roles`, which `extract` calls)."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -1491,7 +1611,11 @@ class Store:
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets) - ("seniority" in sets)
|
||||
remote = str(fields.get("remote") or "")
|
||||
where = [str(name) for name in fields.get("remote_where") or []]
|
||||
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
|
||||
sets["remote_model"] = remote
|
||||
sets["remote_where_model"] = json.dumps(where)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["place_fold"] = places.fold(sets["place"])
|
||||
@@ -1499,8 +1623,8 @@ class Store:
|
||||
gazetteer.texts({"place": sets["place"]}))
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
(*sets.values(), uid))
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
|
||||
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
|
||||
return filled
|
||||
|
||||
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
||||
@@ -2129,6 +2253,18 @@ class Store:
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
||||
"GROUP BY place, country")]
|
||||
|
||||
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
|
||||
"""`(area, open roles)` for every remote area an open remote role
|
||||
holds, the area as place ids in order: what `places.suggest` counts
|
||||
the remote tokens from. Copies are not counted, as in the feed."""
|
||||
found: dict[str, list[int]] = {}
|
||||
for r in self._rows(
|
||||
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
|
||||
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
|
||||
"ORDER BY rr.uid, rr.rowid"):
|
||||
found.setdefault(str(r[0]), []).append(int(r[1]))
|
||||
return list(Counter(tuple(ids) for ids in found.values()).items())
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`, copies
|
||||
included: what a board or a search holds, not what the feed lists."""
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26]
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26]
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27]
|
||||
# The tag of 21 is All countries and All roles since 24.
|
||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||
|
||||
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||
conn.execute("PRAGMA user_version = 22")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [23, 24, 25, 26, 27]
|
||||
assert migrated.agents()[0].fetch_terms == []
|
||||
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
|
||||
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
||||
conn.execute("PRAGMA user_version = 23")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [24, 25, 26]
|
||||
assert migrated.migrations_applied == [24, 25, 26, 27]
|
||||
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
||||
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
||||
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
||||
|
||||
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 26
|
||||
assert LATEST == 27
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [25, 26]
|
||||
assert migrated.migrations_applied == [25, 26, 27]
|
||||
with sqlite3.connect(path) as conn:
|
||||
found = {sid: rest for sid, *rest in conn.execute(
|
||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||
@@ -748,12 +748,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [26]
|
||||
assert migrated.migrations_applied == [26, 27]
|
||||
role = migrated.role(posting("a").uid)
|
||||
assert role is not None and role.posting_url == ""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||
assert Store(path).migrations_applied == [26]
|
||||
assert Store(path).migrations_applied == [26, 27]
|
||||
|
||||
|
||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||
|
||||
Reference in new issue
Block a user