Merge the remote area: remote tokens, role_remote, remote_from, model wins for remote (wp-remote-a)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> # Conflicts: # app/models.py
This commit is contained in:
commit
e5572cab9d
21 files changed
+1250
-143
No files matched your search
+58
-4
@@ -118,7 +118,9 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
prep_due INTEGER NOT NULL DEFAULT 0,
|
||||
remote_model TEXT NOT NULL DEFAULT '',
|
||||
remote_where_model TEXT
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
@@ -129,9 +131,12 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
-- three levels behind roles.seniority: the model's (extract), the host's own
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
|
||||
-- among the last columns: SQLite cannot drop a column a comment precedes,
|
||||
-- which the migration tests do.
|
||||
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
|
||||
-- extract's remote, which wins over the host's word and the place line, and
|
||||
-- remote_where_model the names it read for where a remote hire may live (a
|
||||
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
|
||||
-- columns: SQLite cannot drop a column a comment precedes, which the
|
||||
-- migration tests do.
|
||||
|
||||
-- The gazetteer: a place is a continent, a country (key the ISO code), a
|
||||
-- region or a city, in the place parent_id names. key is unique per kind:
|
||||
@@ -177,6 +182,15 @@ CREATE TABLE IF NOT EXISTS role_places (
|
||||
url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- A remote role's area, the places a remote hire may live in (app/prep/area.py),
|
||||
-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with
|
||||
-- no row is not remote, or remote with its area not stated.
|
||||
CREATE TABLE IF NOT EXISTS role_remote (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (uid, place_id)
|
||||
);
|
||||
|
||||
-- The open vocabulary summarise reads off a posting: the role family, the
|
||||
-- technologies and skills it names, and the domain. What the feed's filter
|
||||
-- row is built from; see Store.set_tags and Store.tag_counts.
|
||||
@@ -424,6 +438,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The remote roles whose area holds a set of places (a remote: token).
|
||||
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
@@ -629,6 +645,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
# A Python step: it adds the column only where it is missing.
|
||||
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
|
||||
"fetched", lambda conn: _add_posting_url(conn)),
|
||||
# A Python step: it adds the columns only where they are missing, and seeds
|
||||
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
|
||||
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
|
||||
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -878,6 +898,40 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||
|
||||
|
||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 27's step: `roles.remote_model` empty and
|
||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||
roles it has read once more for the area; `role_remote`, empty until
|
||||
the next prep; and Anywhere, seeded with the rest of the gazetteer.
|
||||
|
||||
Prep now writes `remote` outright from the model, the host's word and
|
||||
the place line, where it used to keep a filled column. A board role's
|
||||
`remote` without a host word becomes its `remote_host`, which a pull
|
||||
keeps until the host states another word: on live 1,280 of the 3,880
|
||||
open ones are given by nothing prep reads now, mostly Greenhouse, Oracle
|
||||
and Workday roles an older reader mapped. A search role's `remote_host`
|
||||
its reader no longer sends is an older reading of its title ("...
|
||||
Remote Sensing ...", 41 roles), and is cleared for prep to read the
|
||||
title again. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
if "remote_model" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''")
|
||||
if "remote_where_model" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT")
|
||||
kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' "
|
||||
"AND remote != '' AND source != 'search'").rowcount
|
||||
cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' "
|
||||
"AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL"
|
||||
).rowcount
|
||||
log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d",
|
||||
kept, cleared)
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS role_remote (
|
||||
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
|
||||
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (uid, place_id))""")
|
||||
gazetteer.seed(conn)
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
+175
-39
@@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
#: structured field wins over what a model read out of the prose, and the next
|
||||
#: upsert from that board would put it back anyway. `function` and
|
||||
#: `seniority` are not here either, for the opposite reason — see
|
||||
#: upsert from that board would put it back anyway. `function`,
|
||||
#: `seniority` and `remote` are not here, for the opposite reason — see
|
||||
#: `set_extracted`.
|
||||
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
|
||||
|
||||
#: How many replies may leave a role unanswered before `extract` stops
|
||||
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
|
||||
@@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
|
||||
GIVE_UP_AFTER = 3
|
||||
|
||||
#: A role `extract` has yet to read: still open, no copy of another
|
||||
#: (`rebuild_dupes`), not given up, and either never answered or without a
|
||||
#: model level. The second half makes `extract` the afterburner for the level:
|
||||
#: a role read before the level was asked for is read again, and only fills
|
||||
#: what is still empty (`set_extracted`). A reply that names no level leaves
|
||||
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
|
||||
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
|
||||
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
#: (`rebuild_dupes`), not given up, and either never answered, without a
|
||||
#: model level, or never asked where a remote hire may live
|
||||
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
|
||||
#: for the level and the area: a role read before they were asked for is
|
||||
#: read again, and only fills what is still empty (`set_extracted`). A reply
|
||||
#: that names no level leaves the role waiting and is charged a try
|
||||
#: (`extract.apply_batch`), so it ends.
|
||||
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
|
||||
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
|
||||
|
||||
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
|
||||
VOTES = (-1, 0, 1, None)
|
||||
@@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray:
|
||||
return np.frombuffer(blob, dtype=_DTYPE)
|
||||
|
||||
|
||||
#: A role with no place the gazetteer resolved, over `roles r`.
|
||||
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
|
||||
def _unplaced(gaz: places.Gazetteer) -> str:
|
||||
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
|
||||
is no place for this, should a person ever alias it."""
|
||||
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
|
||||
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
|
||||
|
||||
|
||||
#: A remote role whose area is stated (`role_remote`), over `roles r`.
|
||||
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
|
||||
|
||||
|
||||
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
|
||||
"""A remote role whose area holds one of these places, over `table`."""
|
||||
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
|
||||
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
|
||||
|
||||
|
||||
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
@@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
filter does (`_places_sql`): a country gate is met by a role whose
|
||||
`country` is one of its codes or that has a place in one of them, so
|
||||
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
||||
role has neither. A place gate's value is a place token, bare text a
|
||||
`place:`."""
|
||||
role has neither. A `require` gate on country reads a remote role whose
|
||||
area is stated through the area alone, as the remote chips do
|
||||
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
|
||||
meet Europe, "Remote US" does not, and neither is empty. A place gate's
|
||||
value is a place token, bare text a `place:`.
|
||||
|
||||
`remote_from` takes place tokens without their `remote:` (`country:DK`,
|
||||
`anywhere`) and matches as the remote chips do. It is empty for a role
|
||||
whose remote is unknown, and for a remote role with no area stated; a
|
||||
hybrid or onsite role fails it."""
|
||||
column = f"r.{gate.field}"
|
||||
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
|
||||
marks = ", ".join("?" for _ in values) or "NULL"
|
||||
if gate.field == "salary_min":
|
||||
floor = min(_int_or_none(v) or 0 for v in gate.values)
|
||||
return f"{column} IS NULL", f"{column} >= ?", [floor]
|
||||
if gate.field == "remote_from":
|
||||
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
|
||||
match, params = _places_sql(tokens, gaz, table="r")
|
||||
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
|
||||
if gate.field == "country":
|
||||
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
|
||||
codes=[v.upper() for v in values])
|
||||
return f"(r.country = '' AND {_UNPLACED})", match, params
|
||||
tokens = [f"country:{v}" for v in values]
|
||||
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
|
||||
empty = f"(r.country = '' AND {_unplaced(gaz)})"
|
||||
if gate.mode != "require":
|
||||
return empty, match, params
|
||||
named = {p for t in tokens for p in gaz.lookup(t) or ()}
|
||||
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
|
||||
return (f"({empty} AND NOT {_HAS_AREA})",
|
||||
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
|
||||
if gate.field == "place":
|
||||
match, params = _places_sql(values, gaz, table="r")
|
||||
return "r.place = ''", match, params
|
||||
@@ -238,7 +271,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
|
||||
|
||||
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
||||
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
|
||||
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
|
||||
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
||||
role with a place under one of the places the tokens name (`country:DK`
|
||||
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
||||
@@ -246,12 +279,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
||||
role whose board gave a country and no place). `codes` are more values of
|
||||
`country` to accept. A `place:` token the gazetteer does not know is
|
||||
matched on whole words of the role's folded line (`places.match_sql`),
|
||||
so a town nobody has entered yet is still found."""
|
||||
so a town nobody has entered yet is still found.
|
||||
|
||||
A `remote:` token is a remote role whose area overlaps its places
|
||||
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
|
||||
`remote:any` is every remote role that meets or leaves empty each of
|
||||
`scope`, the viewing agent's `require` gates on country; with none,
|
||||
every remote role."""
|
||||
ids: set[int] = set()
|
||||
areas: set[int] = set()
|
||||
found = {c for c in codes if c}
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
remote: list[tuple[str, list[Any]]] = []
|
||||
for token in tokens:
|
||||
tok = places.parse(token)
|
||||
if tok.kind == "remote":
|
||||
if tok.value == "any":
|
||||
remote.append(_in_scope(scope, gaz, table))
|
||||
elif named_area := gaz.lookup(token):
|
||||
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
|
||||
continue
|
||||
named = gaz.lookup(token)
|
||||
if named is None:
|
||||
sql, values = places.match_sql(token, place=f"{table}.place_fold",
|
||||
@@ -270,9 +318,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
||||
clauses.insert(1 if ids else 0,
|
||||
f"{table}.country IN (SELECT value FROM json_each(?))")
|
||||
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
||||
if areas:
|
||||
remote.append(_areas_sql(areas, table))
|
||||
for sql, values in remote:
|
||||
clauses.append(sql)
|
||||
params += values
|
||||
return f"({' OR '.join(clauses) or '0'})", params
|
||||
|
||||
|
||||
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
|
||||
"""`remote:any` over `table`: a remote role that meets or leaves empty
|
||||
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
|
||||
place is in every scope."""
|
||||
if not scope:
|
||||
return f"{table}.remote = 'remote'", []
|
||||
parts: list[str] = []
|
||||
params: list[Any] = []
|
||||
for gate in scope:
|
||||
empty, match, values = _gate_sql(gate, gaz)
|
||||
parts.append(f"({empty} OR {match})")
|
||||
params += values
|
||||
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
|
||||
f"AND {' AND '.join(parts)})", params)
|
||||
|
||||
|
||||
def _known_places(tokens: Sequence[str]) -> None:
|
||||
"""ValueError for a token that names nothing, which would otherwise match
|
||||
no role without saying why."""
|
||||
@@ -804,6 +873,7 @@ class Store:
|
||||
role.tags = self._tags_of(uid)
|
||||
role.copies = self._copies_of(uid)
|
||||
role.places = self.role_places(uid)
|
||||
role.remote_where = self._remote_where([uid]).get(uid, "")
|
||||
return role
|
||||
|
||||
def _copies_of(self, uid: str) -> list[RoleCopy]:
|
||||
@@ -828,6 +898,25 @@ class Store:
|
||||
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
|
||||
"GROUP BY dup_of", (json.dumps(list(uids)),))}
|
||||
|
||||
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the area as a label` ("Europe", "United States, Canada",
|
||||
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
|
||||
with an area stated; a role without one is absent. See
|
||||
`Role.remote_where`."""
|
||||
if not uids:
|
||||
return {}
|
||||
found: dict[str, list[int]] = {}
|
||||
for r in self._rows(
|
||||
"""SELECT rr.uid, rr.place_id FROM role_remote rr
|
||||
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
|
||||
WHERE rr.uid IN (SELECT value FROM json_each(?))
|
||||
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
|
||||
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
|
||||
if not found:
|
||||
return {}
|
||||
gaz = self.gazetteer()
|
||||
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
|
||||
|
||||
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
|
||||
"""`uid -> the line the feed shows` for a page of roles that have
|
||||
more than one place (`dupes.place_line`); a role with one place or
|
||||
@@ -1032,8 +1121,10 @@ class Store:
|
||||
found = [ScoredRole.from_row(r) for r in self._rows(
|
||||
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
areas = self._remote_where([r.uid for r in found])
|
||||
for r in found:
|
||||
r.tags = by_uid.get(r.uid, [])
|
||||
r.remote_where = areas.get(r.uid, "")
|
||||
return found
|
||||
|
||||
def roles_page(self, limit: int = 100, company: str | None = None,
|
||||
@@ -1089,10 +1180,12 @@ class Store:
|
||||
by_uid = self._tags_by_uid([f.uid for f in found])
|
||||
copies = self._copy_counts([f.uid for f in found])
|
||||
lines = self._place_lines([f.uid for f in found])
|
||||
areas = self._remote_where([f.uid for f in found])
|
||||
for f in found:
|
||||
f.tags = by_uid.get(f.uid, [])
|
||||
f.copies = copies.get(f.uid, 0)
|
||||
f.place = lines.get(f.uid, f.place)
|
||||
f.remote_where = areas.get(f.uid, "")
|
||||
return found
|
||||
|
||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||
@@ -1226,7 +1319,7 @@ class Store:
|
||||
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
|
||||
if places:
|
||||
_known_places(places)
|
||||
clause, values = _places_sql(places, self.gazetteer())
|
||||
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
|
||||
where.append(clause)
|
||||
params += values
|
||||
sql = f"FROM roles {_WITH_VOTES}"
|
||||
@@ -1240,6 +1333,13 @@ class Store:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
return sql, [*joins, *params]
|
||||
|
||||
def _scope(self, agent_id: int | None) -> list[Gate]:
|
||||
"""The agent's `require` gates on country: what `remote:any` keeps a
|
||||
remote role within. None without an agent."""
|
||||
found = self.agent(agent_id) if agent_id is not None else None
|
||||
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
|
||||
if found is not None else []
|
||||
|
||||
def tag_counts(self, agent_id: int | None = None,
|
||||
show_muted: bool = False) -> list[tuple[str, int, int]]:
|
||||
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
||||
@@ -1305,17 +1405,27 @@ class Store:
|
||||
|
||||
# ----------------------------------------------------------------- prep
|
||||
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
|
||||
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
|
||||
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||
due (`prep_due` above 0), every open role, or every role."""
|
||||
due (`prep_due` above 0), every open role, or every role; of `uids`
|
||||
only, when given. `remote_area` is the role's `role_remote` as a JSON
|
||||
list, in order."""
|
||||
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||
params: list[Any] = []
|
||||
if uids is not None:
|
||||
where += " AND uid IN (SELECT value FROM json_each(?))"
|
||||
params.append(json.dumps(list(uids)))
|
||||
return self._rows(
|
||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country,
|
||||
json_extract(raw, '$.places') AS raw_places
|
||||
FROM roles WHERE {where}""")
|
||||
employment, remote, country, remote_model, remote_where_model,
|
||||
json_extract(raw, '$.places') AS raw_places,
|
||||
(SELECT json_group_array(place_id) FROM (
|
||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||
ORDER BY rr.rowid)) AS remote_area
|
||||
FROM roles WHERE {where}""", params)
|
||||
|
||||
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve these roles' own places against the gazetteer, writing the
|
||||
@@ -1336,26 +1446,31 @@ class Store:
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||
`seniority_title`, `employment`, `remote`, `remote_area` and
|
||||
`country`. `seniority` is the model's level, else the stated, else
|
||||
the title's. An empty `remote` or `country` leaves the column as it
|
||||
is, since `extract` fills an empty one. `remote_places`, what the
|
||||
place texts say, only fills an empty column: it is a guess from a
|
||||
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||
changed the role since it was read. Returns the rows written."""
|
||||
the title's. `remote` is written as prep composed it (the model's,
|
||||
else the host's word, else the place line) and `remote_area`, place
|
||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||
column as it is, since `extract` fills an empty one. `prep_due` goes
|
||||
back to 0 only when no pull changed the role since it was read.
|
||||
Returns the rows written."""
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
written = conn.executemany(
|
||||
"""UPDATE roles SET company_key = :company_key,
|
||||
seniority_stated = :seniority_stated,
|
||||
seniority_title = :seniority_title,
|
||||
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||
:remote_places),
|
||||
remote = :remote,
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
WHERE uid = :uid""", mapped).rowcount
|
||||
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
|
||||
[(m["uid"],) for m in mapped])
|
||||
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
|
||||
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
|
||||
return written
|
||||
|
||||
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||
"""`(field, host, word, open roles)` for every host word an open role
|
||||
@@ -1425,7 +1540,8 @@ class Store:
|
||||
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
|
||||
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
|
||||
.replace("dup_of", "r.dup_of")
|
||||
.replace("seniority_model", "r.seniority_model")]
|
||||
.replace("seniority_model", "r.seniority_model")
|
||||
.replace("remote_where_model", "r.remote_where_model")]
|
||||
if uids is not None:
|
||||
where.append("r.uid IN (SELECT value FROM json_each(?))")
|
||||
return " AND ".join(where), [json.dumps(list(uids))]
|
||||
@@ -1475,7 +1591,11 @@ class Store:
|
||||
the rule that then keeps a later pull from putting the guess back.
|
||||
`seniority` is the other: a non-empty answer is `seniority_model`,
|
||||
which comes before the stated and the title's level, so it becomes
|
||||
`seniority` too."""
|
||||
`seniority` too. `remote` and `remote_where` are written outright as
|
||||
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
|
||||
even when empty, since an answer is what the model read this time;
|
||||
`remote` and the area are composed from them by prep, so the role is
|
||||
marked due (`prep.run_roles`, which `extract` calls)."""
|
||||
with self.transaction() as conn:
|
||||
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
|
||||
if row is None:
|
||||
@@ -1498,7 +1618,11 @@ class Store:
|
||||
languages = [str(code) for code in fields.get("languages") or []]
|
||||
if languages and json.loads(row["languages"] or "[]") == []:
|
||||
sets["languages"] = json.dumps(languages)
|
||||
filled = len(sets) - ("seniority" in sets)
|
||||
remote = str(fields.get("remote") or "")
|
||||
where = [str(name) for name in fields.get("remote_where") or []]
|
||||
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
|
||||
sets["remote_model"] = remote
|
||||
sets["remote_where_model"] = json.dumps(where)
|
||||
if "place" in sets:
|
||||
sets["place_key"] = place_key(sets["place"])
|
||||
sets["place_fold"] = places.fold(sets["place"])
|
||||
@@ -1506,8 +1630,8 @@ class Store:
|
||||
gazetteer.texts({"place": sets["place"]}))
|
||||
sets["extracted_at"] = self.now()
|
||||
conn.execute(
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
|
||||
(*sets.values(), uid))
|
||||
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
|
||||
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
|
||||
return filled
|
||||
|
||||
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
|
||||
@@ -2141,6 +2265,18 @@ class Store:
|
||||
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
||||
"GROUP BY place, country")]
|
||||
|
||||
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
|
||||
"""`(area, open roles)` for every remote area an open remote role
|
||||
holds, the area as place ids in order: what `places.suggest` counts
|
||||
the remote tokens from. Copies are not counted, as in the feed."""
|
||||
found: dict[str, list[int]] = {}
|
||||
for r in self._rows(
|
||||
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
|
||||
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
|
||||
"ORDER BY rr.uid, rr.rowid"):
|
||||
found.setdefault(str(r[0]), []).append(int(r[1]))
|
||||
return list(Counter(tuple(ids) for ids in found.values()).items())
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`, copies
|
||||
included: what a board or a search holds, not what the feed lists."""
|
||||
|
||||
Reference in new issue
Block a user