Merge the remote area: remote tokens, role_remote, remote_from, model wins for remote (wp-remote-a)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

# Conflicts:
#	app/models.py
This commit is contained in:
Wolf committed 2026-09-30 16:21:57 +02:00
commit e5572cab9d
21 files changed
+1250 -143

No files matched your search

+40 -7
View File
@@ -7,9 +7,11 @@ Sharing the backlog out a call at a time, the give-up count, splitting a
refused batch, and failing with a notice when no model is reachable are
`desk.modeltask`'s.
`extract` is also the afterburner for the level. A role is waiting while it is
open, has not been given up on, has never been answered (`extracted_at IS
NULL`) or has no model level (`seniority_model` empty), and is in the best of an enabled agent's
`extract` is also the afterburner for the level and the remote area. A role
is waiting while it is open, has not been given up on, has never been
answered (`extracted_at IS NULL`), has no model level (`seniority_model`
empty) or was never asked where a remote hire may live (`remote_where_model`
NULL), and is in the best of an enabled agent's
ungated scores as deep and as low as that agent's own `limits` say
(`extract_top`, `extract_min`). That bound is what keeps the cost
proportional to the rankings rather than to the boards:
@@ -20,7 +22,10 @@ fills columns that are still empty: a board's structured field wins over what
a model read out of the prose. The exception is the level, which the model
always gives (the board's word and the title's go in as hints) and which
overrules them; both stay in `seniority_stated` and `seniority_title`, so
`Store.seniority_agreement` can compare them. See `Store.set_extracted`.
`Store.seniority_agreement` can compare them. Remote is the other: the
model's `remote` and `remote_where` are kept as they come, and prep composes
the role's `remote` (the model's first) and its area from them
(`prep.run_roles`) before the rescore reads them. See `Store.set_extracted`.
Off by default (`extract.enabled: false`), because on the API backend every
batch is a paid call. Whether it runs is `stages.stage_on`: the switch under
@@ -39,6 +44,7 @@ from typing import Any
from desk import modeltask
from desk.modeltask import ModelTask
from pipeline import NULL, RunContext, Task, job
from app import prep
from app.config import Config
from app.jobs import stages
from app.models import LEVELS, PERMITS, REMOTE, SENIORITIES, ExtractResult, Role
@@ -72,6 +78,15 @@ Fields:
("Berlin", "Greater London"). '' if none is named.
- country: ISO 3166-1 alpha-2 code of that place ("DE", "GB"), or ''.
- remote: "remote", "hybrid" or "onsite" only when the posting says so, else ''.
"remote" when the hire works from home most of the time, even with a
regular office week (one week a month, a few days a quarter); "hybrid" when
there are set office days every week; "onsite" when the work is at the
office or the site.
- remote_where: where a remote hire may live, as the posting states it: the
countries, regions or continents by their English names (["Denmark"],
["Europe"], ["United States", "Canada"]), or ["anywhere"] for worldwide.
An office location is not this, only where the hire may work from. [] when
the role is not remote or the posting does not say.
- seniority: exactly one of {", ".join(LEVELS)}, for every posting; never ''.
This is the one field where you decide rather than extract. The levels are
one ladder; from staff up each rung has an individual-contributor level and
@@ -119,6 +134,7 @@ SCHEMA: dict[str, Any] = {
"place": _TEXT,
"country": _TEXT,
"remote": {"type": "string", "enum": list(REMOTE)},
"remote_where": {"type": "array", "items": _TEXT},
"seniority": {"type": "string", "enum": list(LEVELS)},
"salary_min": _SALARY,
"salary_max": _SALARY,
@@ -127,7 +143,7 @@ SCHEMA: dict[str, Any] = {
"permit": {"type": "string", "enum": list(PERMITS)},
"languages_required": {"type": "array", "items": _TEXT},
},
"required": ["uid", "place", "country", "remote", "seniority",
"required": ["uid", "place", "country", "remote", "remote_where", "seniority",
"salary_min", "salary_max", "currency", "function",
"permit", "languages_required"],
"additionalProperties": False,
@@ -186,15 +202,28 @@ def _salary(value: Any) -> int | None:
return int(value) if value > 0 else None
#: How many names `remote_where` keeps, and how long each may be.
_WHERE_NAMES, _WHERE_LEN = 12, 60
def _names(value: Any) -> list[str]:
"""`remote_where` checked: strings, whitespace collapsed, each once."""
found = [" ".join(str(name).split())[:_WHERE_LEN]
for name in (value if isinstance(value, list) else [])]
return list(dict.fromkeys(name for name in found if name))[:_WHERE_NAMES]
def fields(answer: dict[str, Any]) -> dict[str, Any]:
"""One answer, checked field by field. Anything outside a field's
vocabulary is '' or None, which `set_extracted` never writes."""
vocabulary is '' or None, which `set_extracted` never writes, except
for `remote` and `remote_where`, which it writes as they come."""
languages = [code for raw in answer.get("languages_required") or []
if (code := _code(raw, _LANGUAGE, upper=False))]
return {
"place": " ".join(str(answer.get("place") or "").split())[:200],
"country": _code(answer.get("country"), _COUNTRY, upper=True),
"remote": _pick(answer.get("remote"), REMOTE),
"remote_where": _names(answer.get("remote_where")),
"seniority": _pick(answer.get("seniority"), SENIORITIES),
"salary_min": _salary(answer.get("salary_min")),
"salary_max": _salary(answer.get("salary_max")),
@@ -211,8 +240,10 @@ def apply_batch(store: Store, answer: dict[str, Any],
"""Write the answers back. A uid the batch did not carry is rejected
rather than written, so a reply cannot reach a role it was not shown. A
role the reply leaves out keeps `extracted_at` NULL and is charged a
turn by `modeltask.one`."""
turn by `modeltask.one`. The roles answered are mapped again
(`prep.run_roles`), so their remote and area follow the model's."""
wanted = {row["uid"] for row in (batch or {}).get("roles", [])}
answered: list[str] = []
stats = ExtractResult()
for one in answer.get("roles") or []:
uid = str(one.get("uid") or "")
@@ -228,8 +259,10 @@ def apply_batch(store: Store, answer: dict[str, Any],
stats.rejected.append(uid)
continue
wanted.discard(uid)
answered.append(uid)
stats.extracted += 1
stats.filled += filled
prep.run_roles(store, answered)
return stats
+9 -5
View File
@@ -6,17 +6,19 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
- **Base reach**: the union of the enabled agents' `any` gates on country,
place (a town or region, mapped to its country through the gazetteer) and
remote. An agent whose `any` group is empty, or holds any other member (a
function, a remote value other than `remote`, a place the gazetteer does
not know), can take a role from anywhere, so the base reach is then
everywhere.
remote; a `remote_from` gate is remote. An agent whose `any` group is
empty, or holds any other member (a function, a remote value other than
`remote`, a place the gazetteer does not know), can take a role from
anywhere, so the base reach is then everywhere.
- **Wide reach, for the companies set to All countries**
(`companies.all_countries`, global): the union over the enabled agents of
their `require` reach, the intersection of each one's `require` gates on
country and place, Europe today, expanded to its countries through the
gazetteer, and the base reach, so the override never reads less than the
board would without it. An agent with no `require` gate on place makes it
everywhere. A role in Freiburg may be open to remote, or worth asking the
everywhere. A `require` gate on remote or `remote_from` next to one on
country adds nothing to it, so remote never widens it past the countries.
A role in Freiburg may be open to remote, or worth asking the
company about for Denmark. Favourite (affinity 2) is scoring only, where
it skips the `any` gates; it does not widen what is fetched.
- **Board override**: a board whose subscription `params` name a country, a
@@ -145,6 +147,8 @@ def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach:
everywhere for a gate that says nothing of place or names one the
gazetteer does not know."""
values = [str(v).strip() for v in gate.values if str(v).strip()]
if gate.field == "remote_from":
return Reach(remote=True)
if gate.field == "remote":
return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \
else EVERYWHERE
+12 -3
View File
@@ -970,6 +970,10 @@ class Role(Model):
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
#: Where a remote hire may live, as a label: "anywhere", "Europe",
#: "United States, Canada". '' when the role is not remote or its area is
#: not stated. Not a column: filled by `Store` from `role_remote`.
remote_where: str = ""
@computed_field # type: ignore[prop-decorator]
@property
@@ -1076,6 +1080,9 @@ class RoleRow(Model):
excerpt: str = ""
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
copies: int = 0
#: As `Role.remote_where`: the area a remote hire may live in, '' when
#: the role is not remote or does not say.
remote_where: str = ""
@computed_field # type: ignore[prop-decorator]
@property
@@ -1170,11 +1177,13 @@ class Application(Model):
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title`
#: matches a value as whole words anywhere in the title, folded as `title_key`
#: folds it, so `account executive` finds "Enterprise Account Executive, UK"
#: and `data engineer` does not find "Data Engineering Manager"; the rest are
#: and `data engineer` does not find "Data Engineering Manager"; `remote_from`
#: takes place tokens (`country:DK`, `country:EU`, `anywhere`) and is met by a
#: remote role whose area overlaps one, as a `remote:` chip is; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company", "title"]
GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function",
"salary_min", "currency", "permit", "languages", "company", "title"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer", "exclude"]
+249 -27
View File
@@ -26,6 +26,21 @@ against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`),
and a token is the places under the place it names. `matches` stays the rule
for a `place:` token the gazetteer does not know. This module opens no
database; the caller hands over the places it holds, or its connection.
A fourth token finds remote roles by where their hire may live:
remote:country:DK a remote role whose area (`role_remote`, written by
`app.prep`) overlaps Denmark: Denmark or a place in
it, a place Denmark lies in (Europe, Øresund), or
Anywhere
remote:anywhere a remote role whose area is Anywhere, and no other
remote:any every remote role in the viewing agent's scope, its
`require` gates on country (`Store`, which knows the
agent, answers it)
Anywhere is a place of its own, outside the tree (`ANYWHERE`), so `under`
never reaches every role and the Place filter stays a filter. It has no
alias: only a remote area holds it, never a role's places.
"""
from __future__ import annotations
@@ -33,16 +48,17 @@ from __future__ import annotations
import re
import sqlite3
from collections import Counter
from collections.abc import Iterable
from collections.abc import Callable, Iterable, Sequence
from dataclasses import dataclass
from functools import lru_cache
from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold,
country_of)
country_of, remote_of)
__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed",
"Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse",
"placeless", "register", "seed", "suggest", "towns"]
__all__ = ["ANYWHERE", "ANYWHERE_WORDS", "KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place",
"Region", "Seed", "Suggestion", "Token", "fold", "known", "label", "match_sql",
"matches", "parse", "placeless", "register", "remote_part", "seed", "suggest",
"towns"]
def fold(text: str) -> str:
@@ -409,7 +425,8 @@ _KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN)
@dataclass(frozen=True)
class Token:
"""One place filter. `value` is a country code, a region key or a fold."""
"""One place filter. `value` is a country code, a region key or a fold;
for `remote`, the place token (`country:DK`), `anywhere` or `any`."""
kind: str
value: str
@@ -424,12 +441,36 @@ def _region_key(text: str) -> str:
return _REGION_BY_NAME.get(folded, folded)
#: The words that are Anywhere: in a remote token (`remote:anywhere`), and in
#: a place text that says remote ("Home Based - Worldwide").
ANYWHERE_WORDS = frozenset(("anywhere", "worldwide", "world wide"))
def _remote(value: str) -> Token:
"""`remote:<place token>`: `value` is the place token, whose label the
chip reads as "Remote · Denmark"; Europe (`country:EU`) is "Remote ·
Europe", `anywhere` "Remote · anywhere" and `any` "Remote". A remote
token inside one names nothing."""
if fold(value) == "any":
return Token("remote", "any", "Remote")
if fold(value) in ANYWHERE_WORDS:
return Token("remote", "anywhere", "Remote · anywhere")
inner = parse(value)
if inner.kind == "remote" or not value:
return Token("remote", "", "Remote")
shown = "Europe" if (inner.kind, inner.value) == ("country", "EU") else inner.label
return Token("remote", str(inner), f"Remote · {shown}")
def parse(token: str) -> Token:
"""`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that
is none of these, is a place: `Skåne` is `place:skane`. A country by name
(`country:Denmark`) and a region by label (`region:Skåne`) are read as well."""
"""`country:DK`, `region:skane`, `place:malmo` or `remote:<one of these>`.
Bare text, or a prefix that is none of these, is a place: `Skåne` is
`place:skane`. A country by name (`country:Denmark`) and a region by label
(`region:Skåne`) are read as well."""
kind, sep, value = (token or "").partition(":")
kind = kind.strip().lower()
if sep and kind == "remote":
return _remote(value.strip())
if not sep or kind not in ("country", "region", "place"):
kind, value = "place", token or ""
value = value.strip()
@@ -447,8 +488,9 @@ def parse(token: str) -> Token:
def label(token: str | Token) -> str:
"""The text of a chip read from the address: `country:DK` is `Denmark`,
`region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not
know is its fold in title case."""
`region:skane` is `Skåne`, `place:malmo` is `Malmö`, `remote:country:DK`
is `Remote · Denmark`. A town the tables do not know is its fold in title
case."""
return (token if isinstance(token, Token) else parse(token)).label
@@ -459,12 +501,15 @@ SQL_FUNCTION = "place_match"
def known(token: str) -> bool:
"""Whether the token names something: a country of the table (or `EU`), a
region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ`
and `place:` are not."""
region of `REGIONS`, a place with a value, or `remote:` and one of these,
`anywhere` or `any`. `region:atlantis`, `country:ZZ`, `place:` and
`remote:country:ZZ` are not."""
return _known(parse(token))
def _known(tok: Token) -> bool:
if tok.kind == "remote":
return tok.value in ("any", "anywhere") or (bool(tok.value) and _known(parse(tok.value)))
if tok.kind == "country":
return tok.value in _COUNTRY_LABELS or tok.value == "EU"
if tok.kind == "region":
@@ -492,7 +537,7 @@ def _matcher(token: str) -> _Matcher | None:
`place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate
written as a bare word means what the owner meant by it."""
tok = parse(token)
if not _known(tok):
if not _known(tok) or tok.kind == "remote":
return None
phrases: set[str] = set()
countries: list[tuple[str, frozenset[str]]] = []
@@ -525,7 +570,8 @@ def matches(place_fold: str, country: str, token: str) -> bool:
token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or
a Danish city) while `country` is empty or `EU`. `region:skane` is any of
its names or towns as whole words, `place:malmo` the town and its aliases.
A token that names nothing (`known` is false) matches no role."""
A token that names nothing (`known` is false) matches no role, nor does a
`remote:` token, which only the gazetteer's areas answer."""
match = _matcher(token)
if match is None:
return False
@@ -561,7 +607,7 @@ def match_sql(token: str, *, place: str = "place_fold", country: str = "country"
with OR by the caller. Bare text is a place. A token that names nothing
(`known` is false) is `0`, which matches no role."""
tok = parse(token)
if not _known(tok):
if not _known(tok) or tok.kind == "remote":
return "0", []
return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)]
@@ -571,8 +617,8 @@ def match_sql(token: str, *, place: str = "place_fold", country: str = "country"
@dataclass(frozen=True)
class Suggestion:
"""`token` is what the chip holds, `kind` is `country`, `region` or `place`,
`count` the open roles it stands for."""
"""`token` is what the chip holds, `kind` is `country`, `region`, `place`
or `remote`, `count` the open roles it stands for."""
token: str
label: str
@@ -603,12 +649,24 @@ _DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$")
_LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$")
def remote_part(text: str) -> bool:
"""Whether a place text, or one part of it, says the role is worked
remotely: "Germany (Remote)", "Remote - US", "Home Based - EMEA",
"Worldwide". One that says hybrid too is not ("Remote or hybrid,
Berlin"), as `remote_of` reads hybrid first."""
return remote_of(text) == "remote"
def _towns(place: str) -> list[str]:
"""The first comma part of each place of a line: `Aarhus N, Region Central
Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal
district is dropped, since `place:kobenhavn` finds `København K` anyway."""
district is dropped, since `place:kobenhavn` finds `København K` anyway.
A part that says remote (`remote_part`) names where a hire may live, not
a town: `USA (Remote)` and `Remote Germany` give none."""
found = []
for part in _PART.split(_AND_HOME.sub("", place or "")):
if remote_part(part):
continue
head = part.split(",")[0].strip()
if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN:
head = lead[2]
@@ -663,17 +721,77 @@ def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[s
return counts
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
-> list[Suggestion]:
#: The kinds a remote suggestion is offered for by name alone; a region or a
#: town only when a remote area names it or a place in it.
_WIDE = ("continent", "country")
def _asks_remote(text: str) -> tuple[bool, str]:
"""Whether typed text asks for the remote tokens as such, and the text of
a place after "remote": `remo` is `(True, "")`, `remote den` `(True,
"den")`, `den` `(False, "den")`. Three letters at least, so "re" is
still Regensburg."""
head, _, rest = text.partition(" ")
if head == "remote":
return True, rest.strip()
return (True, "") if len(text) >= 3 and "remote".startswith(text) else (False, text)
def _remote_suggestions(text: str, gaz: Gazetteer,
areas: list[tuple[tuple[int, ...], int]]) -> list[Suggestion]:
"""`remote:` tokens for typed text, each with the open remote roles it
finds (`Gazetteer.overlap`, which holds Anywhere), none that finds
nothing, widest first and then by open roles. A place is offered when it
is a continent or a country, or a remote area names it or a place in it
(`held`). "remote" alone offers every continent and country held;
"remote den" or "den" the places offered whose name it starts, a region
or a town only when its token names it alone. `remote:anywhere` is a
token and no suggestion: every remote chip holds Anywhere already."""
asked, text = _asks_remote(text)
held: set[int] = set()
for ids, _ in areas:
for place_id in ids:
held |= {place_id, *gaz.ancestors(place_id)}
if asked and not text:
candidates = {p for p in held if gaz.places[p].kind in _WIDE}
elif text:
candidates = {p for p, place in gaz.places.items()
if (place.kind in _WIDE or p in held) and _prefixes(text, gaz.names(p))}
else:
candidates = set()
found: dict[str, tuple[int, Suggestion]] = {}
for place_id in candidates - {gaz.anywhere}:
token = parse(f"remote:{gaz.token(place_id)}")
named = gaz.lookup(str(token))
if str(token) in found or not named or place_id not in named \
or (token.value.startswith("place:") and len(named) > 1):
continue
wanted = gaz.overlap(named)
if count := sum(n for ids, n in areas if not wanted.isdisjoint(ids)):
found[str(token)] = (KINDS.index(gaz.places[place_id].kind),
Suggestion(str(token), token.label, "remote", count))
return [s for _, s in sorted(found.values(), key=lambda f: (f[0], -f[1].count, f[1].label))]
def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12,
remote: Iterable[tuple[Iterable[int], int]] = (),
gaz: Gazetteer | None = None,
remote_any: Callable[[], int] | None = None) -> list[Suggestion]:
"""Tokens for the text typed so far. `places` is `(place, country, open
roles)`. Countries first, then regions, then towns, each by open roles.
roles)`, `remote` is `(area, open roles)` for the open remote roles, the
area as place ids of `gaz`, and `remote_any` counts the open roles
`remote:any` finds for the agent asking, called only when it is offered.
`remote:any` ("Remote") first when `q` starts "remote", then countries,
then the other remote tokens, then regions, then towns, each by open
roles.
A country is suggested when `q` starts a word of one of its names, with the
roles that carry its code. A region likewise on its label and names, with the
roles whose line is in it. A town is the first comma part of each `;` part of
a line, grouped by fold and by alias (`Copenhagen` and `København` are one
town, shown as its commonest spelling). Countries, regions and towns with no
open role are left out: a chip that finds nothing helps nobody."""
open role are left out: a chip that finds nothing helps nobody. Remote
tokens are `_remote_suggestions`, offered only with `gaz`."""
text = fold(q)
if not text:
return []
@@ -719,10 +837,16 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
else _TOWN_LABELS.get(group) or group.title()
towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total))
remotes = _remote_suggestions(text, gaz, [(tuple(ids), int(n)) for ids, n in remote]) \
if gaz is not None else []
first = []
if remote_any is not None and _asks_remote(text)[0] and (count := remote_any()):
first.append(Suggestion("remote:any", "Remote", "remote", count))
countries.sort(key=lambda s: (-s.count, s.label))
regions.sort(key=lambda s: (-s.count, s.label))
towns.sort(key=lambda s: (-s.count, s.label))
return (countries + regions + towns)[:limit]
return (first + countries + remotes + regions + towns)[:limit]
# ------------------------------------------------------------------ gazetteer
@@ -730,6 +854,15 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \
#: The kinds of place, widest first: the index is how specific a kind is.
KINDS: tuple[str, ...] = ("continent", "country", "region", "city")
#: `(kind, key)` of Anywhere, where a remote hire may live when a posting says
#: "worldwide". A continent with no parent and nothing in it, so `under` never
#: reaches every role.
ANYWHERE = ("continent", "world")
#: What is Anywhere in a part of a place text that says remote: "Remote,
#: Global" is Anywhere, while "Global HQ, Berlin" is Berlin.
_ANYWHERE_IF_REMOTE = frozenset(("global", "international", *ANYWHERE_WORDS))
#: Entries of `_CITIES` that name a region rather than a town.
_CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia",
"bay area", "emilia romagna", "tuscany", "lazio"))
@@ -788,7 +921,7 @@ def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
regions of `REGIONS`; the towns of the alias groups, of the regions and
of `_CITIES`, one place per group (`Copenhagen`, `København`,
`Kopenhamn` are one), in their region when a region lists them and else
in their country."""
in their country. Anywhere (`ANYWHERE`) stands apart, with no alias."""
europe = ("continent", "europe")
seeds: dict[tuple[str, str], Seed] = {}
@@ -806,6 +939,7 @@ def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]:
# Asia Pacific holds no country: a role placed there alone keeps the
# host's country, and one beside a country is that country.
add("continent", "asia-pacific", "Asia Pacific", None, ("asia pacific", "apac"))
add(*ANYWHERE, "Anywhere", None, ())
for code, names in _COUNTRY_LABELS.items():
add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code])
for country, code, name in _subdivisions():
@@ -892,6 +1026,9 @@ class Gazetteer:
self._link(whole, part)
self._ancestors: dict[int, frozenset[int]] = {}
self._resolved: dict[str, tuple[int, ...]] = {}
self._overlap: dict[frozenset[int], frozenset[int]] = {}
self._names: dict[int, tuple[str, ...]] | None = None
self.anywhere: int | None = self.by_key.get(ANYWHERE)
def _link(self, whole: int, part: int) -> None:
self._up.setdefault(part, set()).add(whole)
@@ -921,6 +1058,85 @@ class Gazetteer:
todo += self._down.get(place_id, ())
return frozenset(seen)
def overlap(self, ids: Iterable[int]) -> frozenset[int]:
"""Every place a remote area may hold to overlap these: the places,
what lies under them, every place one of those lies in, and Anywhere.
Denmark's holds Europe, and Øresund through the Capital Region, so a
role remote within Europe is open to someone in Denmark."""
key = frozenset(ids)
if (found := self._overlap.get(key)) is None:
under = self.under(key)
wider = set(under)
for place_id in under:
wider |= self.ancestors(place_id)
if self.anywhere is not None:
wider.add(self.anywhere)
found = self._overlap[key] = frozenset(wider)
return found
def names(self, place_id: int) -> tuple[str, ...]:
"""The folds a place is written as: its aliases and its name."""
if self._names is None:
by_id: dict[int, dict[str, None]] = {}
for folded, found in self.aliases.items():
for one in found:
by_id.setdefault(one, {})[folded] = None
for place in self.places.values():
by_id.setdefault(place.id, {})[fold(place.name)] = None
self._names = {place_id: tuple(folds) for place_id, folds in by_id.items()}
return self._names.get(place_id, ())
def token(self, place_id: int) -> str:
"""The filter token that names a place: `country:DK`, `country:EU`
for Europe, `region:skane` for a region of `REGIONS`, `anywhere` for
Anywhere, else `place:` and the name folded."""
place = self.places[place_id]
if place_id == self.anywhere:
return "anywhere"
if (place.kind, place.key) == ("continent", "europe"):
return "country:EU"
if place.kind == "country":
return f"country:{place.key}"
if place.kind == "region" and place.key in REGIONS:
return f"region:{place.key}"
return f"place:{fold(place.name)}"
def remote_places(self, text: str, marked: bool = False) -> tuple[int, ...]:
"""The places the parts of a place text that say remote
(`remote_part`) name, where a remote hire may live: `Germany
(Remote)` is Germany, `Cardiff, London or Remote (UK)` the United
Kingdom, `Home Based - Worldwide` and `Remote, Global` Anywhere.
`marked` takes every part, for a name the model read as where a
remote hire may live ("Europe", "anywhere").
`global` and `international` are Anywhere only here, so `Global HQ,
Berlin` stays Berlin; beside another place Anywhere is dropped, so
`Anywhere in France, Belgium` is the two countries. `Remote` alone
names none."""
found: list[int] = []
for part in _PART.split(_AND_HOME.sub("", text or "")):
if not (marked or remote_part(part)):
continue
ids = [p for p in self.resolve(part) if p != self.anywhere]
if self.anywhere is not None and any(
f" {word} " in f" {fold(part)} " for word in _ANYWHERE_IF_REMOTE):
ids.append(self.anywhere)
if len(ids) > 1 and self.anywhere in ids:
ids.remove(self.anywhere)
found += ids
return tuple(dict.fromkeys(found))
def area_label(self, ids: Sequence[int]) -> str:
"""A remote area as a row shows it: the names in order, Anywhere as
"anywhere", "United States, Canada". More than three places are
shown as the countries they lie in, so twenty German states read
"Germany"."""
found = [p for p in ids if p in self.places]
if len(found) > 3:
found = list(dict.fromkeys(
self.by_key.get(("country", self.country(p)), p) for p in found))
return ", ".join("anywhere" if p == self.anywhere else self.places[p].name
for p in found)
def country(self, place_id: int) -> str:
"""The ISO code of the country a place is in, '' above or outside one."""
place: Place | None = self.places.get(place_id)
@@ -1014,9 +1230,15 @@ class Gazetteer:
"""The places a filter token names, before what lies in them: `country:DK`
is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo`
every place written so (a town, or a region or a country by name).
None when the gazetteer knows no such place."""
A `remote:` token names the places of its place token, `remote:anywhere`
Anywhere, `remote:any` none. None when the gazetteer knows no such
place."""
tok = parse(token)
if tok.kind == "country":
if tok.kind == "remote":
if tok.value != "anywhere":
return self.lookup(tok.value) if tok.value not in ("", "any") else None
found = {self.anywhere} if self.anywhere is not None else set()
elif tok.kind == "country":
key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value)
found = {self.by_key[key]} if key in self.by_key else set()
elif tok.kind == "region":
+5 -3
View File
@@ -7,16 +7,18 @@ and embedding. One door: `from app.prep import ...`.
words.py seniority, employment and remote: the host's table, the
vocabulary itself, the shared remote words
country.py the country from the resolved places, else the host's
area.py a remote role's area: where a remote hire may live
hosts/ one table file per reader (`hosts.TABLES`)
stage.py `run`: the stage over the due roles, `unmapped` and the fold
stage.py `run`: the stage over the due roles, `unmapped` and the fold;
`run_roles`: a few roles at once, for `extract`
"""
from __future__ import annotations
from app.prep.country import country
from app.prep.hosts import TABLES
from app.prep.stage import STATE, Prepped, company, fingerprint, run
from app.prep.stage import STATE, Prepped, company, fingerprint, run, run_roles
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
"employment", "fingerprint", "lookup", "remote", "run", "run_roles", "seniority"]
+55
View File
@@ -0,0 +1,55 @@
"""A remote role's area: the places a remote hire may live in.
`area` takes the first of these that names a place:
1. the model's `remote_where`, the names `extract` read in the posting,
each read as a place text that says remote ("Europe", "anywhere")
2. the parts of the role's place texts that say remote in words
(`Gazetteer.remote_places`): "Germany (Remote)", "Home Based - EMEA",
"Remote, Global"
3. the role's resolved places, each as its country: Databricks' "Denmark"
with the host's remote flag is Denmark, and Airtame's "København,
Hovedstaden, Denmark; Remote" too, since an office town says where the
company is, not that the hire must live in it. A place the reader
marked remote without the word is the host's flag, and read so.
"Remote" alone names no place, so a role with nothing else has no area: it
is remote, area not stated. The area is kept as its widest places
(`widest`), in the order found.
"""
from __future__ import annotations
from collections.abc import Iterable, Sequence
from app import places
def widest(gaz: places.Gazetteer, ids: Iterable[int]) -> tuple[int, ...]:
"""The places no other one of them lies in, in order, each once:
Germany and Europe is Europe. Anywhere alone when it is one of them,
since it holds every other."""
found = tuple(dict.fromkeys(ids))
if gaz.anywhere in found:
return (gaz.anywhere,) if gaz.anywhere is not None else ()
return tuple(p for p in found if not any(q in found for q in gaz.ancestors(p)))
def _country(gaz: places.Gazetteer, place_id: int) -> int:
"""The country a place lies in, else the place itself (a continent,
Øresund)."""
return gaz.by_key.get(("country", gaz.country(place_id)), place_id)
def area(gaz: places.Gazetteer, texts: Sequence[str], place_ids: Sequence[int],
model: Sequence[str] | None = None) -> tuple[int, ...]:
"""The area of a remote role. `texts` are its place texts, `place_ids`
its resolved places, `model` the names `extract` read (None never asked,
`[]` not stated)."""
for found in (
[p for name in model or () for p in gaz.remote_places(name, marked=True)],
[p for text in texts for p in gaz.remote_places(text)],
[_country(gaz, p) for p in place_ids]):
if found:
return widest(gaz, found)
return ()
+53 -16
View File
@@ -3,14 +3,17 @@
A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), and its country
from the places (`app.prep.country`). When the tables or the gazetteer in
code (`places.seed`) changed since the last run (`fingerprint`, kept in
state as `prep:tables`), the new places and spellings are added to the
database and every open role is mapped again from its stored words, with
no new pull. Then the words no
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
copies across the whole database.
and remote through the host's tables (`app.prep.words`), its country from
the places (`app.prep.country`), and a remote role's area
(`app.prep.area`). `remote` is the model's (`remote_model`), else the
host's word, else what the place line or a search role's title says. When
the tables or the gazetteer in code (`places.seed`) changed since the last
run (`fingerprint`, kept in state as `prep:tables`), the new places and
spellings are added to the database and every open role is mapped again
from its stored words, with no new pull. Then the words no table knows are
listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the
whole database. `run_roles` maps a few roles outside a run, for `extract`,
whose model answer changes remote and area.
"""
from __future__ import annotations
@@ -24,14 +27,16 @@ from app import places
from app.models import company_key
from app.models import company_fold
from app.prep import words
from app.prep.area import area
from app.prep.country import country, of_host
from app.prep.hosts import TABLES
from app.prep.words import Tables
from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes.
VERSION = 2
#: so every open role is mapped again as it is after a table changes. 3: the
#: remote area, and remote as the model's before the host's word.
VERSION = 3
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
@@ -75,11 +80,30 @@ def _places(raw: Any) -> list[Any]:
return found if isinstance(found, list) else []
def _texts(row: Mapping[str, Any]) -> list[str]:
"""The role's place texts: `raw`'s places, else the line cut at `;`."""
given = _places(row["raw_places"])
return [words.place_text(p) for p in given] if given \
else str(row["place"] or "").split(";")
def _model_names(raw: Any) -> list[str] | None:
"""`remote_where_model` read: None never asked, else the names."""
if raw is None:
return None
return [str(name) for name in _places(raw)]
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
host = str(row["host"] or "")
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
texts = _texts(row)
remote = row["remote_model"] \
or words.remote(host, row["remote_host"] or None, (), "", tables) \
or words.remote(host, None, _places(row["raw_places"]), row["place"], tables,
title=row["title"] if row["source"] == "search" else "")
return {
"uid": row["uid"],
"due": row["prep_due"],
@@ -87,10 +111,9 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
"seniority_stated": stated,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
"remote_places": words.remote(
host, None, _places(row["raw_places"]), row["place"], tables,
title=row["title"] if row["source"] == "search" else ""),
"remote": remote,
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
if remote == "remote" else ()),
"country": country(gaz, place_ids, row["country_host"]),
}
@@ -98,8 +121,8 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or (found["remote"] and found["remote"] != row["remote"]) \
or (found["remote_places"] and not row["remote"]) \
or found["remote"] != row["remote"] \
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
or (found["country"] and found["country"] != row["country"]) \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
@@ -143,3 +166,17 @@ def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
copies = store.rebuild_dupes().copies if dupes else None
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
copies=copies)
def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int:
"""Map these roles now, due or not, and nothing else: what `extract`
calls for the roles it answered, so the rescore that follows sees the
model's remote and area. Returns the rows written."""
if not uids:
return 0
rows = store.prep_rows("all", uids=uids)
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables)
for r in rows])
+12 -5
View File
@@ -52,6 +52,12 @@ _REMOTE_LINE = re.compile(
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
#: "Remote" in a title that names a field or a product, not how the role is
#: worked: "Remote Sensing Specialist", "Remote Control Engineer".
_NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\b",
re.IGNORECASE)
def _line_word(text: str) -> str:
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
if found := remote_of(text):
@@ -98,7 +104,8 @@ def employment(host: str, word: str,
return lookup("employment", host, word, tables) or ""
def _place_text(place: Any) -> str:
def place_text(place: Any) -> str:
"""A place of `raw`'s `places`, a dict with `text` or a string, as its text."""
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
@@ -114,17 +121,17 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
nothing, the place line or the place texts are read for a word
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
`title` when given, which the stage gives for a search provider's role
only ("Backend Developer - Remote"), and last a place the reader marked
remote is remote."""
only ("Backend Developer - Remote", but not "Remote Sensing Analyst"),
and last a place the reader marked remote is remote."""
if value is True or value == "true":
return "remote"
said = value not in (None, False, "false")
if said and (found := lookup("remote", host, str(value), tables)):
return found
places = list(places)
if found := _line_word(line or "; ".join(map(_place_text, places))):
if found := _line_word(line or "; ".join(map(place_text, places))):
return found
if title and (found := _line_word(title)):
if title and (found := _line_word(_NOT_REMOTE_WORK.sub(" ", title))):
return found
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
return "remote"
+58 -4
View File
@@ -118,7 +118,9 @@ CREATE TABLE IF NOT EXISTS roles (
remote_host TEXT NOT NULL DEFAULT '',
country_host TEXT NOT NULL DEFAULT '',
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0
prep_due INTEGER NOT NULL DEFAULT 0,
remote_model TEXT NOT NULL DEFAULT '',
remote_where_model TEXT
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
@@ -129,9 +131,12 @@ CREATE TABLE IF NOT EXISTS roles (
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
-- three levels behind roles.seniority: the model's (extract), the host's own
-- word mapped to app.seniority's vocabulary, and the title's. No comment sits
-- among the last columns: SQLite cannot drop a column a comment precedes,
-- which the migration tests do.
-- word mapped to app.seniority's vocabulary, and the title's. remote_model is
-- extract's remote, which wins over the host's word and the place line, and
-- remote_where_model the names it read for where a remote hire may live (a
-- JSON list, NULL never asked, [] not stated). No comment sits among the last
-- columns: SQLite cannot drop a column a comment precedes, which the
-- migration tests do.
-- The gazetteer: a place is a continent, a country (key the ISO code), a
-- region or a city, in the place parent_id names. key is unique per kind:
@@ -177,6 +182,15 @@ CREATE TABLE IF NOT EXISTS role_places (
url TEXT NOT NULL DEFAULT ''
);
-- A remote role's area, the places a remote hire may live in (app/prep/area.py),
-- in the order found. Anywhere (places.ANYWHERE) is one of them. A role with
-- no row is not remote, or remote with its area not stated.
CREATE TABLE IF NOT EXISTS role_remote (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (uid, place_id)
);
-- The open vocabulary summarise reads off a posting: the role family, the
-- technologies and skills it names, and the domain. What the feed's filter
-- row is built from; see Store.set_tags and Store.tag_counts.
@@ -424,6 +438,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
-- The remote roles whose area holds a set of places (a remote: token).
CREATE INDEX IF NOT EXISTS idx_role_remote_place ON role_remote(place_id, uid);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
"""
@@ -629,6 +645,10 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# A Python step: it adds the column only where it is missing.
(26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is "
"fetched", lambda conn: _add_posting_url(conn)),
# A Python step: it adds the columns only where they are missing, and seeds
# Anywhere. The areas are written by the next prep (app.prep.stage.VERSION 3).
(27, "roles.remote_model and roles.remote_where_model, role_remote and the Anywhere "
"place: where a remote hire may live", lambda conn: _add_remote_area(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -878,6 +898,40 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
def _add_remote_area(conn: sqlite3.Connection) -> None:
"""Migration 27's step: `roles.remote_model` empty and
`roles.remote_where_model` NULL for every role, so `extract` reads the
roles it has read once more for the area; `role_remote`, empty until
the next prep; and Anywhere, seeded with the rest of the gazetteer.
Prep now writes `remote` outright from the model, the host's word and
the place line, where it used to keep a filled column. A board role's
`remote` without a host word becomes its `remote_host`, which a pull
keeps until the host states another word: on live 1,280 of the 3,880
open ones are given by nothing prep reads now, mostly Greenhouse, Oracle
and Workday roles an older reader mapped. A search role's `remote_host`
its reader no longer sends is an older reading of its title ("...
Remote Sensing ...", 41 roles), and is cleared for prep to read the
title again. Runs twice without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
if "remote_model" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_model TEXT NOT NULL DEFAULT ''")
if "remote_where_model" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_where_model TEXT")
kept = conn.execute("UPDATE roles SET remote_host = remote WHERE remote_host = '' "
"AND remote != '' AND source != 'search'").rowcount
cleared = conn.execute("UPDATE roles SET remote_host = '' WHERE source = 'search' "
"AND remote_host != '' AND json_extract(raw, '$.remote') IS NULL"
).rowcount
log.info("migration 27: remote kept as the host's word on %d roles, cleared on %d",
kept, cleared)
conn.execute("""CREATE TABLE IF NOT EXISTS role_remote (
uid TEXT NOT NULL REFERENCES roles(uid) ON DELETE CASCADE,
place_id INTEGER NOT NULL REFERENCES places(id) ON DELETE CASCADE,
PRIMARY KEY (uid, place_id))""")
gazetteer.seed(conn)
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+175 -39
View File
@@ -51,10 +51,10 @@ _RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
#: structured field wins over what a model read out of the prose, and the next
#: upsert from that board would put it back anyway. `function` and
#: `seniority` are not here either, for the opposite reason — see
#: upsert from that board would put it back anyway. `function`,
#: `seniority` and `remote` are not here, for the opposite reason — see
#: `set_extracted`.
_EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
_EXTRACT_TEXT = ("place", "country", "currency", "permit")
#: How many replies may leave a role unanswered before `extract` stops
#: offering it. newsdesk's number, for newsdesk's reason: a row the model
@@ -62,13 +62,15 @@ _EXTRACT_TEXT = ("place", "country", "remote", "currency", "permit")
GIVE_UP_AFTER = 3
#: A role `extract` has yet to read: still open, no copy of another
#: (`rebuild_dupes`), not given up, and either never answered or without a
#: model level. The second half makes `extract` the afterburner for the level:
#: a role read before the level was asked for is read again, and only fills
#: what is still empty (`set_extracted`). A reply that names no level leaves
#: the role waiting and is charged a try (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = (f"(extracted_at IS NULL OR seniority_model = '') AND closed_at IS NULL "
f"AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: (`rebuild_dupes`), not given up, and either never answered, without a
#: model level, or never asked where a remote hire may live
#: (`remote_where_model` NULL). The last two make `extract` the afterburner
#: for the level and the area: a role read before they were asked for is
#: read again, and only fills what is still empty (`set_extracted`). A reply
#: that names no level leaves the role waiting and is charged a try
#: (`extract.apply_batch`), so it ends.
_EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_where_model IS NULL) "
f"AND closed_at IS NULL AND dup_of IS NULL AND extract_tries < {GIVE_UP_AFTER}")
#: What a vote may be: 1 like, 0 neither way, -1 pass, None unjudged.
VOTES = (-1, 0, 1, None)
@@ -191,8 +193,21 @@ def unpack(blob: bytes) -> np.ndarray:
return np.frombuffer(blob, dtype=_DTYPE)
#: A role with no place the gazetteer resolved, over `roles r`.
_UNPLACED = "r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL)"
def _unplaced(gaz: places.Gazetteer) -> str:
"""A role with no place the gazetteer resolved, over `roles r`. Anywhere
is no place for this, should a person ever alias it."""
other = f" AND place_id != {int(gaz.anywhere)}" if gaz.anywhere is not None else ""
return f"r.uid NOT IN (SELECT uid FROM role_places WHERE place_id IS NOT NULL{other})"
#: A remote role whose area is stated (`role_remote`), over `roles r`.
_HAS_AREA = "(r.remote = 'remote' AND r.uid IN (SELECT uid FROM role_remote))"
def _areas_sql(ids: Iterable[int], table: str) -> tuple[str, list[Any]]:
"""A remote role whose area holds one of these places, over `table`."""
return (f"({table}.remote = 'remote' AND {table}.uid IN (SELECT uid FROM role_remote "
"WHERE place_id IN (SELECT value FROM json_each(?))))", [json.dumps(sorted(ids))])
def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
@@ -204,18 +219,36 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
filter does (`_places_sql`): a country gate is met by a role whose
`country` is one of its codes or that has a place in one of them, so
`DK` is met by `Copenhagen; London`. Its column is empty only when the
role has neither. A place gate's value is a place token, bare text a
`place:`."""
role has neither. A `require` gate on country reads a remote role whose
area is stated through the area alone, as the remote chips do
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
meet Europe, "Remote US" does not, and neither is empty. A place gate's
value is a place token, bare text a `place:`.
`remote_from` takes place tokens without their `remote:` (`country:DK`,
`anywhere`) and matches as the remote chips do. It is empty for a role
whose remote is unknown, and for a remote role with no area stated; a
hybrid or onsite role fails it."""
column = f"r.{gate.field}"
values = [str(v).strip().lower() for v in gate.values if str(v).strip()]
marks = ", ".join("?" for _ in values) or "NULL"
if gate.field == "salary_min":
floor = min(_int_or_none(v) or 0 for v in gate.values)
return f"{column} IS NULL", f"{column} >= ?", [floor]
if gate.field == "remote_from":
tokens = [f"remote:{str(v).strip()}" for v in gate.values if str(v).strip()]
match, params = _places_sql(tokens, gaz, table="r")
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
if gate.field == "country":
match, params = _places_sql([f"country:{v}" for v in values], gaz, table="r",
codes=[v.upper() for v in values])
return f"(r.country = '' AND {_UNPLACED})", match, params
tokens = [f"country:{v}" for v in values]
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
empty = f"(r.country = '' AND {_unplaced(gaz)})"
if gate.mode != "require":
return empty, match, params
named = {p for t in tokens for p in gaz.lookup(t) or ()}
area, more = _areas_sql(gaz.overlap(named) if named else (), "r")
return (f"({empty} AND NOT {_HAS_AREA})",
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
if gate.field == "place":
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
@@ -238,7 +271,7 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
codes: Sequence[str] = ()) -> tuple[str, list[Any]]:
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
"""Any of the place tokens over `table`, as a join over the gazetteer: a
role with a place under one of the places the tokens name (`country:DK`
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
@@ -246,12 +279,27 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
role whose board gave a country and no place). `codes` are more values of
`country` to accept. A `place:` token the gazetteer does not know is
matched on whole words of the role's folded line (`places.match_sql`),
so a town nobody has entered yet is still found."""
so a town nobody has entered yet is still found.
A `remote:` token is a remote role whose area overlaps its places
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
`remote:any` is every remote role that meets or leaves empty each of
`scope`, the viewing agent's `require` gates on country; with none,
every remote role."""
ids: set[int] = set()
areas: set[int] = set()
found = {c for c in codes if c}
clauses: list[str] = []
params: list[Any] = []
remote: list[tuple[str, list[Any]]] = []
for token in tokens:
tok = places.parse(token)
if tok.kind == "remote":
if tok.value == "any":
remote.append(_in_scope(scope, gaz, table))
elif named_area := gaz.lookup(token):
areas |= named_area if tok.value == "anywhere" else gaz.overlap(named_area)
continue
named = gaz.lookup(token)
if named is None:
sql, values = places.match_sql(token, place=f"{table}.place_fold",
@@ -270,9 +318,30 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
clauses.insert(1 if ids else 0,
f"{table}.country IN (SELECT value FROM json_each(?))")
params.insert(1 if ids else 0, json.dumps(sorted(found)))
if areas:
remote.append(_areas_sql(areas, table))
for sql, values in remote:
clauses.append(sql)
params += values
return f"({' OR '.join(clauses) or '0'})", params
def _in_scope(scope: Sequence[Gate], gaz: places.Gazetteer, table: str) -> tuple[str, list[Any]]:
"""`remote:any` over `table`: a remote role that meets or leaves empty
each gate of `scope` (`_gate_sql`), so a remote role with no area and no
place is in every scope."""
if not scope:
return f"{table}.remote = 'remote'", []
parts: list[str] = []
params: list[Any] = []
for gate in scope:
empty, match, values = _gate_sql(gate, gaz)
parts.append(f"({empty} OR {match})")
params += values
return (f"{table}.uid IN (SELECT r.uid FROM roles r WHERE r.remote = 'remote' "
f"AND {' AND '.join(parts)})", params)
def _known_places(tokens: Sequence[str]) -> None:
"""ValueError for a token that names nothing, which would otherwise match
no role without saying why."""
@@ -804,6 +873,7 @@ class Store:
role.tags = self._tags_of(uid)
role.copies = self._copies_of(uid)
role.places = self.role_places(uid)
role.remote_where = self._remote_where([uid]).get(uid, "")
return role
def _copies_of(self, uid: str) -> list[RoleCopy]:
@@ -828,6 +898,25 @@ class Store:
f"AND {_NOT_A_PLACE.format(kept='dup_of')} "
"GROUP BY dup_of", (json.dumps(list(uids)),))}
def _remote_where(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the area as a label` ("Europe", "United States, Canada",
"anywhere", `Gazetteer.area_label`) for the remote roles of `uids`
with an area stated; a role without one is absent. See
`Role.remote_where`."""
if not uids:
return {}
found: dict[str, list[int]] = {}
for r in self._rows(
"""SELECT rr.uid, rr.place_id FROM role_remote rr
JOIN roles ON roles.uid = rr.uid AND roles.remote = 'remote'
WHERE rr.uid IN (SELECT value FROM json_each(?))
ORDER BY rr.uid, rr.rowid""", (json.dumps(list(uids)),)):
found.setdefault(str(r["uid"]), []).append(int(r["place_id"]))
if not found:
return {}
gaz = self.gazetteer()
return {uid: gaz.area_label(ids) for uid, ids in found.items()}
def _place_lines(self, uids: Sequence[str]) -> dict[str, str]:
"""`uid -> the line the feed shows` for a page of roles that have
more than one place (`dupes.place_line`); a role with one place or
@@ -1032,8 +1121,10 @@ class Store:
found = [ScoredRole.from_row(r) for r in self._rows(
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
by_uid = self._tags_by_uid([r.uid for r in found])
areas = self._remote_where([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
r.remote_where = areas.get(r.uid, "")
return found
def roles_page(self, limit: int = 100, company: str | None = None,
@@ -1089,10 +1180,12 @@ class Store:
by_uid = self._tags_by_uid([f.uid for f in found])
copies = self._copy_counts([f.uid for f in found])
lines = self._place_lines([f.uid for f in found])
areas = self._remote_where([f.uid for f in found])
for f in found:
f.tags = by_uid.get(f.uid, [])
f.copies = copies.get(f.uid, 0)
f.place = lines.get(f.uid, f.place)
f.remote_where = areas.get(f.uid, "")
return found
def count_roles(self, company: str | None = None, place: str | None = None,
@@ -1226,7 +1319,7 @@ class Store:
raise ValueError(f"tag_mode is any or all, not {tag_mode!r}")
if places:
_known_places(places)
clause, values = _places_sql(places, self.gazetteer())
clause, values = _places_sql(places, self.gazetteer(), scope=self._scope(agent))
where.append(clause)
params += values
sql = f"FROM roles {_WITH_VOTES}"
@@ -1240,6 +1333,13 @@ class Store:
sql += " WHERE " + " AND ".join(where)
return sql, [*joins, *params]
def _scope(self, agent_id: int | None) -> list[Gate]:
"""The agent's `require` gates on country: what `remote:any` keeps a
remote role within. None without an agent."""
found = self.agent(agent_id) if agent_id is not None else None
return [g for g in found.gates if g.field == "country" and g.mode == "require"] \
if found is not None else []
def tag_counts(self, agent_id: int | None = None,
show_muted: bool = False) -> list[tuple[str, int, int]]:
"""`(tag, open_count, unread_count)` over open roles, most-used tag
@@ -1305,17 +1405,27 @@ class Store:
# ----------------------------------------------------------------- prep
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
def prep_rows(self, scope: Literal["due", "open", "all"] = "due",
uids: Sequence[str] | None = None) -> list[sqlite3.Row]:
"""What `app.prep` maps a role from, and what it holds now: the roles
due (`prep_due` above 0), every open role, or every role."""
due (`prep_due` above 0), every open role, or every role; of `uids`
only, when given. `remote_area` is the role's `role_remote` as a JSON
list, in order."""
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
params: list[Any] = []
if uids is not None:
where += " AND uid IN (SELECT value FROM json_each(?))"
params.append(json.dumps(list(uids)))
return self._rows(
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country,
json_extract(raw, '$.places') AS raw_places
FROM roles WHERE {where}""")
employment, remote, country, remote_model, remote_where_model,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
ORDER BY rr.rowid)) AS remote_area
FROM roles WHERE {where}""", params)
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve these roles' own places against the gazetteer, writing the
@@ -1336,26 +1446,31 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_places` and
`seniority_title`, `employment`, `remote`, `remote_area` and
`country`. `seniority` is the model's level, else the stated, else
the title's. An empty `remote` or `country` leaves the column as it
is, since `extract` fills an empty one. `remote_places`, what the
place texts say, only fills an empty column: it is a guess from a
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
changed the role since it was read. Returns the rows written."""
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
column as it is, since `extract` fills an empty one. `prep_due` goes
back to 0 only when no pull changed the role since it was read.
Returns the rows written."""
with self.transaction() as conn:
return conn.executemany(
written = conn.executemany(
"""UPDATE roles SET company_key = :company_key,
seniority_stated = :seniority_stated,
seniority_title = :seniority_title,
seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
:remote_places),
remote = :remote,
country = COALESCE(NULLIF(:country, ''), country),
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
conn.executemany("DELETE FROM role_remote WHERE uid = ?",
[(m["uid"],) for m in mapped])
conn.executemany("INSERT OR IGNORE INTO role_remote (uid, place_id) VALUES (?, ?)",
[(m["uid"], p) for m in mapped for p in m.get("remote_area", ())])
return written
def host_words(self) -> list[tuple[str, str, str, int]]:
"""`(field, host, word, open roles)` for every host word an open role
@@ -1425,7 +1540,8 @@ class Store:
where = [_EXTRACT_WAITING.replace("extracted_at", "r.extracted_at")
.replace("closed_at", "r.closed_at").replace("extract_tries", "r.extract_tries")
.replace("dup_of", "r.dup_of")
.replace("seniority_model", "r.seniority_model")]
.replace("seniority_model", "r.seniority_model")
.replace("remote_where_model", "r.remote_where_model")]
if uids is not None:
where.append("r.uid IN (SELECT value FROM json_each(?))")
return " AND ".join(where), [json.dumps(list(uids))]
@@ -1475,7 +1591,11 @@ class Store:
the rule that then keeps a later pull from putting the guess back.
`seniority` is the other: a non-empty answer is `seniority_model`,
which comes before the stated and the title's level, so it becomes
`seniority` too."""
`seniority` too. `remote` and `remote_where` are written outright as
`remote_model` and `remote_where_model` (a JSON list, `[]` for none),
even when empty, since an answer is what the model read this time;
`remote` and the area are composed from them by prep, so the role is
marked due (`prep.run_roles`, which `extract` calls)."""
with self.transaction() as conn:
row = conn.execute("SELECT * FROM roles WHERE uid=?", (uid,)).fetchone()
if row is None:
@@ -1498,7 +1618,11 @@ class Store:
languages = [str(code) for code in fields.get("languages") or []]
if languages and json.loads(row["languages"] or "[]") == []:
sets["languages"] = json.dumps(languages)
filled = len(sets) - ("seniority" in sets)
remote = str(fields.get("remote") or "")
where = [str(name) for name in fields.get("remote_where") or []]
filled = len(sets) - ("seniority" in sets) + bool(remote) + bool(where)
sets["remote_model"] = remote
sets["remote_where_model"] = json.dumps(where)
if "place" in sets:
sets["place_key"] = place_key(sets["place"])
sets["place_fold"] = places.fold(sets["place"])
@@ -1506,8 +1630,8 @@ class Store:
gazetteer.texts({"place": sets["place"]}))
sets["extracted_at"] = self.now()
conn.execute(
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)} WHERE uid=?",
(*sets.values(), uid))
f"UPDATE roles SET {', '.join(f'{c}=?' for c in sets)}, "
"prep_due = prep_due + 1 WHERE uid=?", (*sets.values(), uid))
return filled
def extract_missed(self, uids: Sequence[str], why: str = "") -> int:
@@ -2141,6 +2265,18 @@ class Store:
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
"GROUP BY place, country")]
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
"""`(area, open roles)` for every remote area an open remote role
holds, the area as place ids in order: what `places.suggest` counts
the remote tokens from. Copies are not counted, as in the feed."""
found: dict[str, list[int]] = {}
for r in self._rows(
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
"ORDER BY rr.uid, rr.rowid"):
found.setdefault(str(r[0]), []).append(int(r[1]))
return list(Counter(tuple(ids) for ids in found.values()).items())
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`, copies
included: what a board or a search holds, not what the feed lists."""
+17 -5
View File
@@ -293,7 +293,7 @@ class PlaceSuggestion(BaseModel):
token: str
label: str
kind: Literal["country", "region", "place"]
kind: Literal["country", "region", "place", "remote"]
count: int
@@ -337,13 +337,25 @@ def suggest_companies(
def suggest_places(
store: StoreDep,
q: str = "",
agent: int | None = None,
limit: Annotated[int, Query(ge=1, le=50)] = 12,
) -> list[PlaceSuggestion]:
"""Place tokens for the text typed so far: countries, then regions, then
towns, each by open roles (`places.suggest`, over every open role's place
line). An empty `q` suggests nothing."""
"""Place tokens for the text typed so far: "Remote" (`remote:any`) when
`q` starts "remote", countries, the remote tokens ("Remote · Denmark"),
regions, then towns, each by open roles (`places.suggest`, over every
open role's place line and every open remote role's area). The count of
`remote:any` is the remote roles in the scope of `agent`, the default
agent when absent, as the feed counts them. An empty `q` suggests
nothing."""
if not places.fold(q):
return []
viewer = _agent_id(store, agent)
return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count)
for s in places.suggest(q, store.open_places(), limit)]
for s in places.suggest(
q, store.open_places(), limit, remote=store.open_remote_areas(),
gaz=store.gazetteer(),
remote_any=lambda: store.count_roles(places=["remote:any"], agent=viewer,
by_score=False))]
@router.get("/places/labels")
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+1 -1
View File
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
+2 -1
View File
@@ -139,7 +139,8 @@ def test_closed_and_answered_roles_are_not_offered(
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
"WHERE source_id = 'closed'")
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"seniority_model = 'senior' WHERE source_id = 'done'")
"seniority_model = 'senior', remote_where_model = '[]' "
"WHERE source_id = 'done'")
model = stub(monkeypatch, lambda roles: [answer(r["uid"]) for r in roles])
modeltask.run(extract.TASK, config, store)
+1 -1
View File
@@ -434,7 +434,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26]
assert migrated.migrations_applied == [23, 24, 25, 26, 27]
assert migrated.agents()[0].fetch_terms == []
+1 -1
View File
@@ -50,7 +50,7 @@ def names(text: str) -> list[str]:
def test_the_seed_holds_the_dictionaries_in_code() -> None:
seeds, includes = seed()
kinds = Counter(s.kind for s in seeds)
assert {s.key for s in seeds if s.kind == "continent"} == {"europe", "asia-pacific"}
assert {s.key for s in seeds if s.kind == "continent"} == {"europe", "asia-pacific", "world"}
assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line)
keys = {(s.kind, s.key) for s in seeds}
assert {("region", key) for key in places.REGIONS} <= keys
+1 -1
View File
@@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26]
assert migrated.migrations_applied == [24, 25, 26, 27]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+14 -12
View File
@@ -205,21 +205,23 @@ def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
assert row(store, "1")["prep_due"] == 2
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
store: Store) -> None:
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "remote")
# The host's own statement does overwrite.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "hybrid"
# The host's own word wins over the line.
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
prep.run(store, tables=TABLES)
assert row(store, "1")["remote"] == "remote"
uid = row(store, "1")["uid"]
with store.transaction() as conn:
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
"WHERE uid = ?", (uid,))
prep.run(store, tables=TABLES, remap=True)
mapped = row(store, "1")
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
("senior", "lead", "onsite")
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
@@ -314,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26]
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+533
View File
@@ -0,0 +1,533 @@
"""Where a remote role may be worked from: the `remote:` tokens, a remote
role's area in prep, the chips and gates that read it, the model's answer and
migration 27. The place texts are the live boards' (docs/plan-remote.md)."""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import pytest
from desk.models import Item
from app import places, prep
from app.jobs import extract, reach
from app.models import Agent, Gate
from app.places import Gazetteer, label, known, parse, seed, suggest
from app.prep import words
from app.storage import Store
#: The require gate on country of the owner's agent: the EU, the EEA, the
#: UK, Switzerland and `EU`.
EUROPE = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
"LI", "NO", "GB", "CH", "EU"]
def gazetteer() -> Gazetteer:
"""The seeded gazetteer, as a fresh file holds it."""
seeds, includes = seed()
ids = {(s.kind, s.key): n for n, s in enumerate(seeds, 1)}
return Gazetteer([(ids[(s.kind, s.key)], s.kind, s.key, s.name,
ids[s.parent] if s.parent else None) for s in seeds],
[(a, ids[(s.kind, s.key)]) for s in seeds for a in s.aliases],
[(ids[w], ids[p]) for w, p in includes])
GAZ = gazetteer()
def names(ids: Any) -> list[str]:
return [GAZ.places[p].name for p in ids]
# ------------------------------------------------------------------ tokens
@pytest.mark.parametrize(("token", "shown", "text"), [
("remote:country:DK", "Remote · Denmark", "remote:country:DK"),
("remote:country:dk", "Remote · Denmark", "remote:country:DK"),
("remote:country:EU", "Remote · Europe", "remote:country:EU"),
("remote:region:oresund", "Remote · Øresund", "remote:region:oresund"),
("remote:place:berlin", "Remote · Berlin", "remote:place:berlin"),
("remote:anywhere", "Remote · anywhere", "remote:anywhere"),
("remote:worldwide", "Remote · anywhere", "remote:anywhere"),
("remote:any", "Remote", "remote:any"),
])
def test_a_remote_token_reads_its_place_token(token: str, shown: str, text: str) -> None:
assert label(token) == shown
assert str(parse(token)) == text
assert known(token)
def test_a_remote_token_of_nothing_is_not_known() -> None:
for token in ("remote:", "remote:country:ZZ", "remote:region:atlantis",
"remote:remote:country:DK"):
assert not known(token), token
def test_a_remote_token_is_answered_by_the_areas_only() -> None:
assert places.match_sql("remote:country:DK") == ("0", [])
assert not places.matches("remote denmark", "DK", "remote:country:DK")
@pytest.mark.parametrize(("text", "expected"), [
("Sweden (Remote)", ["Sweden"]), # Grafana
("Remote - US", ["United States"]),
("Home Based - EMEA", ["Europe"]),
("Home Based - Worldwide", ["Anywhere"]), # Canonical
("Remote, Global", ["Anywhere"]),
("Distributed, Global", ["Anywhere"]),
("Cardiff, London or Remote (UK)", ["United Kingdom"]),
("Anywhere in France, Belgium, Spain", ["France", "Belgium", "Spain"]),
("Remote", []),
("Global HQ, Berlin", []),
("Berlin", []),
("Hybrid, Berlin", []),
])
def test_the_parts_that_say_remote_name_the_area(text: str, expected: list[str]) -> None:
assert names(GAZ.remote_places(text)) == expected
def test_anywhere_is_no_place_of_a_role() -> None:
assert GAZ.resolve("Worldwide") == () and GAZ.resolve("Anywhere") == ()
assert names(GAZ.resolve("Global HQ, Berlin")) == ["Berlin"]
assert GAZ.under((GAZ.anywhere,)) == {GAZ.anywhere}
def test_denmarks_overlap_is_what_lies_in_it_what_it_lies_in_and_anywhere() -> None:
denmark = GAZ.by_key[("country", "DK")]
found = GAZ.overlap((denmark,))
for key in (("continent", "europe"), ("region", "oresund"), ("region", "hovedstaden"),
("city", "copenhagen"), places.ANYWHERE):
assert GAZ.by_key[key] in found, key
for key in (("country", "US"), ("country", "SE"), ("region", "skane")):
assert GAZ.by_key[key] not in found, key
def test_a_remote_line_is_no_town() -> None:
lines = [("USA (Remote)", "US", 70), ("Remote US", "US", 60), ("Germany (Remote)", "DE", 61),
("Remote Germany", "DE", 14), ("Remote - Berlin", "DE", 3), ("Berlin", "DE", 80)]
for q in ("usa", "remote", "germ", "berl"):
assert [s.label for s in suggest(q, lines) if s.kind == "place"] \
== (["Berlin"] if q == "berl" else []), q
assert places.towns("USA (Remote)") == frozenset()
# ---------------------------------------------------------------- the words
def test_remote_sensing_in_a_title_is_no_remote_work() -> None:
for title in ("Senior Sea Ice Scientist with Experience on Remote Sensing",
"Remote Control Engineer", "Remote-controlled Systems Lead",
"Remote Monitoring Specialist"):
assert words.remote("jobindex", None, (), "København Ø", title=title) == "", title
assert words.remote("jobindex", None, (), "Aarhus", title="Developer - Remote") == "remote"
# ------------------------------------------------------------------ the area
@pytest.fixture
def store(tmp_path: Path) -> Iterator[Store]:
yield Store(tmp_path / "roledesk.db")
def posting(source_id: str, place: str = "", *, board: str = "greenhouse",
**raw: Any) -> Item:
item = Item(source_id=f"{board}:{source_id}", location=f"https://boards.example/{source_id}",
title="Backend Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"board": board, "company": "Acme"} | ({"place": place} if place else {})
| raw)
item.source = "ats"
return item
def add(store: Store, source_id: str, place: str = "", **raw: Any) -> str:
item = posting(source_id, place, **raw)
store.upsert_role(item)
return item.uid
#: One role per case of the plan, by what it stands for.
LINES: dict[str, tuple[str, dict[str, Any]]] = {
"grafana": ("Sweden (Remote)", {}),
"canonical": ("Home Based - EMEA; Home Based - Worldwide", {}),
"databricks": ("Denmark", {"remote": True}),
"airtame": ("København, Hovedstaden, Denmark; Remote", {}),
"europe": ("Remote - Europe", {}),
"germany": ("Germany (Remote)", {}),
"us": ("Remote US", {}),
"plain": ("Remote", {}),
"berlin": ("Global HQ, Berlin", {"remote": True}),
"hybrid": ("Hybrid, Copenhagen", {}),
"office": ("Copenhagen", {}),
}
@pytest.fixture
def roles(store: Store) -> dict[str, str]:
found = {key: add(store, key, line, **raw) for key, (line, raw) in LINES.items()}
prep.run(store, dupes=False)
return found
def area(store: Store, uid: str) -> tuple[str, str]:
role = store.role(uid)
assert role is not None
return role.remote, role.remote_where
def test_each_remote_role_gets_the_area_its_places_say(store: Store,
roles: dict[str, str]) -> None:
assert {key: area(store, uid) for key, uid in roles.items()} == {
"grafana": ("remote", "Sweden"),
"canonical": ("remote", "anywhere"), # EMEA and worldwide is worldwide
"databricks": ("remote", "Denmark"), # the host's flag and the place
"airtame": ("remote", "Denmark"), # an office town, read as its country
"europe": ("remote", "Europe"),
"germany": ("remote", "Germany"),
"us": ("remote", "United States"),
"plain": ("remote", ""), # remote, area not stated
"berlin": ("remote", "Germany"),
"hybrid": ("hybrid", ""),
"office": ("", ""),
}
role = store.role(roles["berlin"])
assert role is not None and [p.name for p in role.places] == ["Berlin"]
def test_the_rows_carry_the_area(store: Store, roles: dict[str, str]) -> None:
rows = {r.uid: r.remote_where for r in store.roles_page(limit=100)}
assert rows[roles["canonical"]] == "anywhere" and rows[roles["plain"]] == ""
listed = {r.uid: r.remote_where for r in store.roles(limit=100, by_score=False)}
assert listed == rows
def test_many_places_read_as_their_countries(store: Store) -> None:
uid = add(store, "komma", "Bayern (Remote); Hessen (Remote); Sachsen (Remote); "
"Saarland (Remote); Dänemark (Remote)")
prep.run(store, dupes=False)
assert area(store, uid) == ("remote", "Germany, Denmark")
def uids(store: Store, *tokens: str, agent: int | None = None) -> set[str]:
return {r.uid for r in store.roles(limit=1000, places=list(tokens), agent=agent,
by_score=False)}
def pick(roles: dict[str, str], *keys: str) -> set[str]:
return {roles[k] for k in keys}
def test_remote_denmark_is_every_remote_role_open_to_someone_in_denmark(
store: Store, roles: dict[str, str]) -> None:
assert uids(store, "remote:country:DK") == pick(
roles, "canonical", "databricks", "airtame", "europe")
assert uids(store, "remote:region:hovedstaden") == uids(store, "remote:country:DK")
def test_remote_europe_is_every_remote_role_in_europe(store: Store,
roles: dict[str, str]) -> None:
assert uids(store, "remote:country:EU") == pick(
roles, "grafana", "canonical", "databricks", "airtame", "europe", "germany", "berlin")
assert uids(store, "remote:country:SE") == pick(
roles, "grafana", "canonical", "europe") # Sweden, anywhere, Europe
def test_remote_anywhere_is_only_anywhere(store: Store, roles: dict[str, str]) -> None:
assert uids(store, "remote:anywhere") == pick(roles, "canonical")
assert uids(store, "remote:country:US") == pick(roles, "canonical", "us")
def test_remote_chips_or_with_the_place_chips(store: Store, roles: dict[str, str]) -> None:
assert uids(store, "remote:anywhere", "place:copenhagen") == pick(
roles, "canonical", "airtame", "hybrid", "office")
def test_remote_any_keeps_the_agents_scope(store: Store, roles: dict[str, str]) -> None:
remote = pick(roles, "grafana", "canonical", "databricks", "airtame", "europe", "germany",
"us", "plain", "berlin")
assert uids(store, "remote:any") == remote
agent = store.save_agent("Europe", "python backend", [
Gate(field="country", values=EUROPE, mode="require")])
assert uids(store, "remote:any", agent=agent.id) == remote - pick(roles, "us")
wide = store.save_agent("Anywhere", "python backend", [
Gate(field="country", values=["DK", "SE"], mode="any")])
assert uids(store, "remote:any", agent=wide.id) == remote
def states(store: Store, *gates: Gate) -> dict[str, list[int]]:
agent = store.save_agent(f"gates {len(store.agents())}", "python backend", list(gates))
return {c.uid: c.gates for c in store.score_candidates(agent.id, list(gates))}
def test_remote_from_is_met_as_the_chip_is_and_empty_without_an_area(
store: Store, roles: dict[str, str]) -> None:
found = states(store, Gate(field="remote_from", values=["country:DK"], mode="any"),
Gate(field="remote_from", values=["anywhere"], mode="any"))
by_key = {key: found[uid] for key, uid in roles.items()}
assert by_key == {
"grafana": [0, 0], "canonical": [1, 1], "databricks": [1, 0], "airtame": [1, 0],
"europe": [1, 0], "germany": [0, 0], "us": [0, 0],
"plain": [-1, -1], # remote, area not stated: neither
"berlin": [0, 0],
"hybrid": [0, 0], # evidence: not remote
"office": [-1, -1], # remote unknown
}
def test_a_require_gate_on_europe_reads_a_remote_role_through_its_area(
store: Store, roles: dict[str, str]) -> None:
found = states(store, Gate(field="country", values=EUROPE, mode="require"))
by_key = {key: found[uid][0] for key, uid in roles.items()}
assert by_key["canonical"] == 1 # Home Based - Worldwide
assert by_key["germany"] == 1
assert by_key["europe"] == 1
assert by_key["plain"] == -1 # "Remote" alone: kept, extract reads it
assert by_key["us"] == 0 # Remote US
assert by_key["office"] == 1 and by_key["hybrid"] == 1
def test_anywhere_aliased_by_hand_is_no_place_for_the_country_gate(store: Store) -> None:
uid = add(store, "ww", "Hybrid, Worldwide")
anywhere = store.gazetteer().anywhere
assert anywhere is not None
store.add_place_alias("Worldwide", anywhere)
prep.run(store, dupes=False)
assert [p.name for p in store.role_places(uid)] == ["Anywhere"]
assert area(store, uid) == ("hybrid", "")
found = states(store, Gate(field="country", values=EUROPE, mode="require"))
assert found[uid] == [-1]
def test_the_owners_agent_keeps_remote_roles_open_to_europe_and_drops_remote_us(
store: Store, roles: dict[str, str]) -> None:
from app.jobs.score import failed_gates
gates = [Gate(field="country", values=EUROPE, mode="require"),
Gate(field="country", values=["DK", "SE"], mode="any"),
Gate(field="remote", values=["remote"], mode="any")]
agent = store.save_agent("Owner", "python backend", gates)
kept = {c.uid for c in store.score_candidates(agent.id, gates) if not failed_gates(c, gates)}
assert pick(roles, "canonical", "germany", "europe", "plain") <= kept
assert roles["us"] not in kept
# ------------------------------------------------------------ the model's say
def test_the_models_remote_wins_and_its_names_are_the_area(store: Store) -> None:
# Black Forest Labs: Ashby says OnSite, the text offers remote work with
# an office week a month.
uid = add(store, "bfl", "Freiburg, Germany", board="ashby", remote="OnSite")
prep.run(store, dupes=False)
assert area(store, uid) == ("onsite", "")
store.set_extracted(uid, {"remote": "remote", "remote_where": []})
prep.run_roles(store, [uid])
assert area(store, uid) == ("remote", "Germany")
store.set_extracted(uid, {"remote": "remote", "remote_where": ["Europe"]})
prep.run_roles(store, [uid])
assert area(store, uid) == ("remote", "Europe")
# A later pull leaves the model's answer standing.
store.upsert_role(posting("bfl", "Freiburg, Germany", board="ashby", remote="OnSite"))
prep.run(store, dupes=False)
assert area(store, uid) == ("remote", "Europe")
def test_set_extracted_writes_the_remote_answer_outright(store: Store) -> None:
uid = add(store, "1", "Berlin")
prep.run(store, dupes=False)
store.set_extracted(uid, {"remote": "remote", "remote_where": ["anywhere"]})
row = store._row("SELECT remote_model, remote_where_model, prep_due FROM roles WHERE uid=?",
(uid,))
assert row is not None
assert (row["remote_model"], json.loads(row["remote_where_model"]), row["prep_due"]) == \
("remote", ["anywhere"], 1)
store.set_extracted(uid, {})
row = store._row("SELECT remote_model, remote_where_model FROM roles WHERE uid=?", (uid,))
assert row is not None and (row["remote_model"], row["remote_where_model"]) == ("", "[]")
@pytest.fixture
def config(tmp_path: Path) -> Any:
from app.config import Config
return Config(database=str(tmp_path / "roledesk.db"), extract={"enabled": True})
def test_extract_reads_the_area_once_more_and_maps_the_role_before_the_rescore(
store: Store, config: Any, monkeypatch: pytest.MonkeyPatch) -> None:
from desk import modeltask
from desk.tools import agent as desk_agent
uid = add(store, "1", "Freiburg, Germany", board="ashby", remote="OnSite")
prep.run(store, dupes=False)
agent = store.save_agent("Backend", "python backend")
store.replace_scores(agent.id, [(uid, 1.0, "")])
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
"seniority_model = 'senior' WHERE uid = ?", (uid,))
assert [r.uid for r in store.extract_waiting(10)] == [uid] # never asked where
def reply(settings: Any, **kwargs: Any) -> desk_agent.Reply:
[one] = kwargs["question"]["roles"]
return desk_agent.Reply(data={"roles": [{
"uid": one["uid"], "place": "", "country": "", "remote": "remote",
"remote_where": ["Denmark", "Sweden"], "seniority": "senior", "salary_min": None,
"salary_max": None, "currency": "", "function": "", "permit": "",
"languages_required": []}]}, backend="api")
monkeypatch.setattr(desk_agent, "backend", lambda settings: "api")
monkeypatch.setattr(desk_agent, "ask_json", reply)
modeltask.run(extract.TASK, config, store)
assert area(store, uid) == ("remote", "Denmark, Sweden")
assert store.extract_waiting(10) == []
def test_the_prompt_asks_where_a_remote_hire_may_live() -> None:
props = extract.SCHEMA["properties"]["roles"]["items"]
assert "remote_where" in props["required"]
assert props["properties"]["remote_where"]["type"] == "array"
assert "office week" in extract.SYSTEM and "remote_where" in extract.SYSTEM
got = extract.fields({"remote": "remote", "remote_where": [" Europe ", "", "Europe", 3]})
assert (got["remote"], got["remote_where"]) == ("remote", ["Europe", "3"])
# ------------------------------------------------------------- suggestions
def test_remote_is_suggested_after_remote_any_and_before_the_regions(
store: Store, roles: dict[str, str]) -> None:
gaz = store.gazetteer()
areas = store.open_remote_areas()
def shown(q: str) -> list[tuple[str, str, str, int]]:
return [(s.kind, s.token, s.label, s.count)
for s in suggest(q, store.open_places(), 20, remote=areas, gaz=gaz,
remote_any=lambda: 9)]
found = shown("remote")
assert found[0] == ("remote", "remote:any", "Remote", 9)
tokens = [t for _, t, _, _ in found]
assert "remote:country:EU" in tokens and "remote:country:US" in tokens
assert "remote:anywhere" not in tokens
assert all(kind == "remote" for kind, _, _, _ in found)
assert ("remote", "remote:country:EU", "Remote · Europe", 7) in found
found = shown("denm")
assert found[:2] == [("country", "country:DK", "Denmark", 4),
("remote", "remote:country:DK", "Remote · Denmark", 4)]
assert shown("remote den")[:2] == [("remote", "remote:any", "Remote", 9),
("remote", "remote:country:DK", "Remote · Denmark", 4)]
assert shown("re") == [t for t in shown("re") if t[1] != "remote:any"]
def test_a_suggested_remote_token_finds_the_roles_it_counted(
store: Store, roles: dict[str, str]) -> None:
gaz = store.gazetteer()
for s in suggest("remote", store.open_places(), 50, remote=store.open_remote_areas(),
gaz=gaz):
assert label(s.token) == s.label
assert len(uids(store, s.token)) == s.count, s.token
def test_the_api_suggests_labels_and_filters_the_remote_tokens(tmp_path: Path) -> None:
from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
store: Store = app.state.store
for key in ("europe", "us", "plain"):
line, raw = LINES[key]
add(store, key, line, **raw)
prep.run(store, dupes=False)
agent = store.save_agent("Europe", "python backend", [
Gate(field="country", values=EUROPE, mode="require")])
with TestClient(app) as client:
found = client.get("/api/suggest/places", params={"q": "remote"}).json()
assert found[0] == {"token": "remote:any", "label": "Remote", "kind": "remote",
"count": 2}
assert {"token": "remote:country:EU", "label": "Remote · Europe", "kind": "remote",
"count": 1} in found
other = client.get("/api/suggest/places", params={"q": "remote", "agent": agent.id})
assert other.json()[0]["count"] == 2
labels = client.get("/api/places/labels", params=[
("tokens", "remote:country:DK"), ("tokens", "remote:anywhere"),
("tokens", "remote:country:ZZ")]).json()
assert labels == [{"token": "remote:country:DK", "label": "Remote · Denmark"},
{"token": "remote:anywhere", "label": "Remote · anywhere"}]
rows = client.get("/api/roles", params={"places": "remote:country:DK"}).json()
assert [r["remote_where"] for r in rows["items"]] == ["Europe"]
assert client.get("/api/roles", params={"places": "remote:country:ZZ"}).status_code \
== 422
# ------------------------------------------------------------------ reach
def test_remote_from_is_remote_reach_and_never_widens_the_require_reach() -> None:
gaz = GAZ
near = Gate(field="remote_from", values=["country:DK"], mode="any")
assert reach.gate_reach(near, gaz) == reach.Reach(remote=True)
owner = Agent(id=2, name="Owner", created_at="2026-09-30", gates=[
Gate(field="country", values=EUROPE, mode="require"),
Gate(field="country", values=["DK", "SE"], mode="any"), near,
Gate(field="remote_from", values=["country:DK"], mode="require")])
assert reach.any_reach(owner, gaz) == reach.Reach(frozenset({"DK", "SE"}), remote=True)
wide = reach.require_reach(owner, gaz)
assert not wide.everywhere and not wide.remote and "DE" in wide.countries
# --------------------------------------------------------------- migration
def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path: Path) -> None:
path = tmp_path / "old.db"
store = Store(path)
board = add(store, "board", "Ireland")
search = Item(source_id="jobindex:1", location="https://jobindex.example/1",
title="Scientist with Experience on Remote Sensing",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"provider": "jobindex", "place": "København Ø", "remote": None})
search.source = "search"
store.upsert_role(search)
with sqlite3.connect(path) as conn:
conn.execute("UPDATE roles SET remote = 'remote', remote_host = '' WHERE uid = ?",
(board,))
conn.execute("UPDATE roles SET remote = 'remote', remote_host = 'remote' "
"WHERE uid = ?", (search.uid,))
conn.execute("DROP INDEX idx_role_remote_place")
conn.execute("DROP TABLE role_remote")
conn.execute("ALTER TABLE roles DROP COLUMN remote_model")
conn.execute("ALTER TABLE roles DROP COLUMN remote_where_model")
conn.execute("DELETE FROM places WHERE kind = 'continent' AND key = 'world'")
conn.execute("PRAGMA user_version = 26")
migrated = Store(path)
assert migrated.migrations_applied == [27]
assert migrated.gazetteer().anywhere is not None
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
"WHERE uid = ?", (board,))
assert row is not None and tuple(row) == ("remote", "", None)
prep.run(migrated, dupes=False)
assert area(migrated, board) == ("remote", "Ireland")
assert area(migrated, search.uid) == ("", "")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 26")
assert Store(path).migrations_applied == [27]
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
uid = add(store, "1", "Berlin")
prep.run(store, dupes=False)
with store.transaction() as conn:
conn.execute("UPDATE roles SET extracted_at = '2026-09-22', seniority_model = 'mid' "
"WHERE uid = ?", (uid,))
assert [r.uid for r in store.extract_waiting(5, [uid])] == [uid]
store.set_extracted(uid, {"seniority": "mid"})
assert store.extract_waiting(5, [uid]) == []
+9 -9
View File
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 26
assert LATEST == 27
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26]
assert migrated.migrations_applied == [25, 26, 27]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -748,12 +748,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26]
assert migrated.migrations_applied == [26, 27]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26]
assert Store(path).migrations_applied == [26, 27]
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
+1 -1
View File
@@ -1227,7 +1227,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posting_url", "posted_at", "score", "gated_reason", "prompt", "vote",
"voted_at", "read_at", "tags", "excerpt", "copies",
"voted_at", "read_at", "tags", "excerpt", "copies", "remote_where",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"