From 189cf813507275091bb0e2ba4817d6a2bb25306c Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 16:00:06 +0200 Subject: [PATCH] Read remote: place tokens, the Anywhere place and a remote area's overlap in places remote:country:DK is a remote role whose area overlaps Denmark: what lies in it, what it lies in, Anywhere. remote:anywhere is Anywhere alone and remote:any the viewing agent's scope. Anywhere is a continent outside the tree with no alias. Remote parts of a line are no town, and the suggestions offer the remote tokens after the countries. Co-Authored-By: Claude Opus 5.5 --- app/places.py | 276 ++++++++++++++++++++++++++++++++++++---- tests/test_gazetteer.py | 2 +- 2 files changed, 250 insertions(+), 28 deletions(-) diff --git a/app/places.py b/app/places.py index a72564a..6f9fe79 100644 --- a/app/places.py +++ b/app/places.py @@ -26,6 +26,21 @@ against a gazetteer (`Gazetteer`, the `places` tables, seeded from `seed`), and a token is the places under the place it names. `matches` stays the rule for a `place:` token the gazetteer does not know. This module opens no database; the caller hands over the places it holds, or its connection. + +A fourth token finds remote roles by where their hire may live: + + remote:country:DK a remote role whose area (`role_remote`, written by + `app.prep`) overlaps Denmark: Denmark or a place in + it, a place Denmark lies in (Europe, Øresund), or + Anywhere + remote:anywhere a remote role whose area is Anywhere, and no other + remote:any every remote role in the viewing agent's scope, its + `require` gates on country (`Store`, which knows the + agent, answers it) + +Anywhere is a place of its own, outside the tree (`ANYWHERE`), so `under` +never reaches every role and the Place filter stays a filter. It has no +alias: only a remote area holds it, never a role's places. """ from __future__ import annotations @@ -33,16 +48,17 @@ from __future__ import annotations import re import sqlite3 from collections import Counter -from collections.abc import Iterable +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass from functools import lru_cache from app.models import (_CITIES, _COUNTRY_NAMES, _EUROPE, _REGIONS, _SUBDIVISIONS, _place_fold, - country_of) + country_of, remote_of) -__all__ = ["KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", "Region", "Seed", - "Suggestion", "Token", "fold", "known", "label", "match_sql", "matches", "parse", - "placeless", "register", "seed", "suggest", "towns"] +__all__ = ["ANYWHERE", "ANYWHERE_WORDS", "KINDS", "REGIONS", "SQL_FUNCTION", "Gazetteer", "Place", + "Region", "Seed", "Suggestion", "Token", "fold", "known", "label", "match_sql", + "matches", "parse", "placeless", "register", "remote_part", "seed", "suggest", + "towns"] def fold(text: str) -> str: @@ -409,7 +425,8 @@ _KNOWN_LONGEST = max(len(name.split()) for name in _KNOWN) @dataclass(frozen=True) class Token: - """One place filter. `value` is a country code, a region key or a fold.""" + """One place filter. `value` is a country code, a region key or a fold; + for `remote`, the place token (`country:DK`), `anywhere` or `any`.""" kind: str value: str @@ -424,12 +441,36 @@ def _region_key(text: str) -> str: return _REGION_BY_NAME.get(folded, folded) +#: The words that are Anywhere: in a remote token (`remote:anywhere`), and in +#: a place text that says remote ("Home Based - Worldwide"). +ANYWHERE_WORDS = frozenset(("anywhere", "worldwide", "world wide")) + + +def _remote(value: str) -> Token: + """`remote:`: `value` is the place token, whose label the + chip reads as "Remote · Denmark"; Europe (`country:EU`) is "Remote · + Europe", `anywhere` "Remote · anywhere" and `any` "Remote". A remote + token inside one names nothing.""" + if fold(value) == "any": + return Token("remote", "any", "Remote") + if fold(value) in ANYWHERE_WORDS: + return Token("remote", "anywhere", "Remote · anywhere") + inner = parse(value) + if inner.kind == "remote" or not value: + return Token("remote", "", "Remote") + shown = "Europe" if (inner.kind, inner.value) == ("country", "EU") else inner.label + return Token("remote", str(inner), f"Remote · {shown}") + + def parse(token: str) -> Token: - """`country:DK`, `region:skane` or `place:malmo`. Bare text, or a prefix that - is none of these, is a place: `Skåne` is `place:skane`. A country by name - (`country:Denmark`) and a region by label (`region:Skåne`) are read as well.""" + """`country:DK`, `region:skane`, `place:malmo` or `remote:`. + Bare text, or a prefix that is none of these, is a place: `Skåne` is + `place:skane`. A country by name (`country:Denmark`) and a region by label + (`region:Skåne`) are read as well.""" kind, sep, value = (token or "").partition(":") kind = kind.strip().lower() + if sep and kind == "remote": + return _remote(value.strip()) if not sep or kind not in ("country", "region", "place"): kind, value = "place", token or "" value = value.strip() @@ -447,8 +488,9 @@ def parse(token: str) -> Token: def label(token: str | Token) -> str: """The text of a chip read from the address: `country:DK` is `Denmark`, - `region:skane` is `Skåne`, `place:malmo` is `Malmö`. A town the tables do not - know is its fold in title case.""" + `region:skane` is `Skåne`, `place:malmo` is `Malmö`, `remote:country:DK` + is `Remote · Denmark`. A town the tables do not know is its fold in title + case.""" return (token if isinstance(token, Token) else parse(token)).label @@ -459,12 +501,15 @@ SQL_FUNCTION = "place_match" def known(token: str) -> bool: """Whether the token names something: a country of the table (or `EU`), a - region of `REGIONS`, a place with a value. `region:atlantis`, `country:ZZ` - and `place:` are not.""" + region of `REGIONS`, a place with a value, or `remote:` and one of these, + `anywhere` or `any`. `region:atlantis`, `country:ZZ`, `place:` and + `remote:country:ZZ` are not.""" return _known(parse(token)) def _known(tok: Token) -> bool: + if tok.kind == "remote": + return tok.value in ("any", "anywhere") or (bool(tok.value) and _known(parse(tok.value))) if tok.kind == "country": return tok.value in _COUNTRY_LABELS or tok.value == "EU" if tok.kind == "region": @@ -492,7 +537,7 @@ def _matcher(token: str) -> _Matcher | None: `place:skane` is Skåne the region, `place:denmark` is Denmark, since a gate written as a bare word means what the owner meant by it.""" tok = parse(token) - if not _known(tok): + if not _known(tok) or tok.kind == "remote": return None phrases: set[str] = set() countries: list[tuple[str, frozenset[str]]] = [] @@ -525,7 +570,8 @@ def matches(place_fold: str, country: str, token: str) -> bool: token. `country:DK` is `country == 'DK'`, or a line that names Denmark (or a Danish city) while `country` is empty or `EU`. `region:skane` is any of its names or towns as whole words, `place:malmo` the town and its aliases. - A token that names nothing (`known` is false) matches no role.""" + A token that names nothing (`known` is false) matches no role, nor does a + `remote:` token, which only the gazetteer's areas answer.""" match = _matcher(token) if match is None: return False @@ -561,7 +607,7 @@ def match_sql(token: str, *, place: str = "place_fold", country: str = "country" with OR by the caller. Bare text is a place. A token that names nothing (`known` is false) is `0`, which matches no role.""" tok = parse(token) - if not _known(tok): + if not _known(tok) or tok.kind == "remote": return "0", [] return f"{SQL_FUNCTION}({place}, {country}, ?)", [str(tok)] @@ -571,8 +617,8 @@ def match_sql(token: str, *, place: str = "place_fold", country: str = "country" @dataclass(frozen=True) class Suggestion: - """`token` is what the chip holds, `kind` is `country`, `region` or `place`, - `count` the open roles it stands for.""" + """`token` is what the chip holds, `kind` is `country`, `region`, `place` + or `remote`, `count` the open roles it stands for.""" token: str label: str @@ -603,12 +649,24 @@ _DISTRICT = re.compile(r"\s+[A-ZÆØÅ]{1,2}$") _LEAD = re.compile(r"^(.+?)\s*[-\u2013]\s*(.+)$") +def remote_part(text: str) -> bool: + """Whether a place text, or one part of it, says the role is worked + remotely: "Germany (Remote)", "Remote - US", "Home Based - EMEA", + "Worldwide". One that says hybrid too is not ("Remote or hybrid, + Berlin"), as `remote_of` reads hybrid first.""" + return remote_of(text) == "remote" + + def _towns(place: str) -> list[str]: """The first comma part of each place of a line: `Aarhus N, Region Central Jutland, Denmark; København, Denmark` is `Aarhus`, `København`. A postal - district is dropped, since `place:kobenhavn` finds `København K` anyway.""" + district is dropped, since `place:kobenhavn` finds `København K` anyway. + A part that says remote (`remote_part`) names where a hire may live, not + a town: `USA (Remote)` and `Remote Germany` give none.""" found = [] for part in _PART.split(_AND_HOME.sub("", place or "")): + if remote_part(part): + continue head = part.split(",")[0].strip() if (lead := _LEAD.match(head)) and fold(lead[1]) in _NOT_A_TOWN: head = lead[2] @@ -663,17 +721,77 @@ def _region_counts(keys: Iterable[str], places: list[tuple[str, int]]) -> dict[s return counts -def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ - -> list[Suggestion]: +#: The kinds a remote suggestion is offered for by name alone; a region or a +#: town only when a remote area names it or a place in it. +_WIDE = ("continent", "country") + + +def _asks_remote(text: str) -> tuple[bool, str]: + """Whether typed text asks for the remote tokens as such, and the text of + a place after "remote": `remo` is `(True, "")`, `remote den` `(True, + "den")`, `den` `(False, "den")`. Three letters at least, so "re" is + still Regensburg.""" + head, _, rest = text.partition(" ") + if head == "remote": + return True, rest.strip() + return (True, "") if len(text) >= 3 and "remote".startswith(text) else (False, text) + + +def _remote_suggestions(text: str, gaz: Gazetteer, + areas: list[tuple[tuple[int, ...], int]]) -> list[Suggestion]: + """`remote:` tokens for typed text, each with the open remote roles it + finds (`Gazetteer.overlap`, which holds Anywhere), none that finds + nothing, widest first and then by open roles. A place is offered when it + is a continent or a country, or a remote area names it or a place in it + (`held`). "remote" alone offers every continent and country held; + "remote den" or "den" the places offered whose name it starts, a region + or a town only when its token names it alone. `remote:anywhere` is a + token and no suggestion: every remote chip holds Anywhere already.""" + asked, text = _asks_remote(text) + held: set[int] = set() + for ids, _ in areas: + for place_id in ids: + held |= {place_id, *gaz.ancestors(place_id)} + if asked and not text: + candidates = {p for p in held if gaz.places[p].kind in _WIDE} + elif text: + candidates = {p for p, place in gaz.places.items() + if (place.kind in _WIDE or p in held) and _prefixes(text, gaz.names(p))} + else: + candidates = set() + found: dict[str, tuple[int, Suggestion]] = {} + for place_id in candidates - {gaz.anywhere}: + token = parse(f"remote:{gaz.token(place_id)}") + named = gaz.lookup(str(token)) + if str(token) in found or not named or place_id not in named \ + or (token.value.startswith("place:") and len(named) > 1): + continue + wanted = gaz.overlap(named) + if count := sum(n for ids, n in areas if not wanted.isdisjoint(ids)): + found[str(token)] = (KINDS.index(gaz.places[place_id].kind), + Suggestion(str(token), token.label, "remote", count)) + return [s for _, s in sorted(found.values(), key=lambda f: (f[0], -f[1].count, f[1].label))] + + +def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12, + remote: Iterable[tuple[Iterable[int], int]] = (), + gaz: Gazetteer | None = None, + remote_any: Callable[[], int] | None = None) -> list[Suggestion]: """Tokens for the text typed so far. `places` is `(place, country, open - roles)`. Countries first, then regions, then towns, each by open roles. + roles)`, `remote` is `(area, open roles)` for the open remote roles, the + area as place ids of `gaz`, and `remote_any` counts the open roles + `remote:any` finds for the agent asking, called only when it is offered. + `remote:any` ("Remote") first when `q` starts "remote", then countries, + then the other remote tokens, then regions, then towns, each by open + roles. A country is suggested when `q` starts a word of one of its names, with the roles that carry its code. A region likewise on its label and names, with the roles whose line is in it. A town is the first comma part of each `;` part of a line, grouped by fold and by alias (`Copenhagen` and `København` are one town, shown as its commonest spelling). Countries, regions and towns with no - open role are left out: a chip that finds nothing helps nobody.""" + open role are left out: a chip that finds nothing helps nobody. Remote + tokens are `_remote_suggestions`, offered only with `gaz`.""" text = fold(q) if not text: return [] @@ -719,10 +837,16 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ else _TOWN_LABELS.get(group) or group.title() towns.append(Suggestion(f"place:{fold(shown)}", shown, "place", total)) + remotes = _remote_suggestions(text, gaz, [(tuple(ids), int(n)) for ids, n in remote]) \ + if gaz is not None else [] + first = [] + if remote_any is not None and _asks_remote(text)[0] and (count := remote_any()): + first.append(Suggestion("remote:any", "Remote", "remote", count)) + countries.sort(key=lambda s: (-s.count, s.label)) regions.sort(key=lambda s: (-s.count, s.label)) towns.sort(key=lambda s: (-s.count, s.label)) - return (countries + regions + towns)[:limit] + return (first + countries + remotes + regions + towns)[:limit] # ------------------------------------------------------------------ gazetteer @@ -730,6 +854,15 @@ def suggest(q: str, places: Iterable[tuple[str, str, int]], limit: int = 12) \ #: The kinds of place, widest first: the index is how specific a kind is. KINDS: tuple[str, ...] = ("continent", "country", "region", "city") +#: `(kind, key)` of Anywhere, where a remote hire may live when a posting says +#: "worldwide". A continent with no parent and nothing in it, so `under` never +#: reaches every role. +ANYWHERE = ("continent", "world") + +#: What is Anywhere in a part of a place text that says remote: "Remote, +#: Global" is Anywhere, while "Global HQ, Berlin" is Berlin. +_ANYWHERE_IF_REMOTE = frozenset(("global", "international", *ANYWHERE_WORDS)) + #: Entries of `_CITIES` that name a region rather than a town. _CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia", "bay area", "emilia romagna", "tuscany", "lazio")) @@ -788,7 +921,7 @@ def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]: regions of `REGIONS`; the towns of the alias groups, of the regions and of `_CITIES`, one place per group (`Copenhagen`, `København`, `Kopenhamn` are one), in their region when a region lists them and else - in their country.""" + in their country. Anywhere (`ANYWHERE`) stands apart, with no alias.""" europe = ("continent", "europe") seeds: dict[tuple[str, str], Seed] = {} @@ -806,6 +939,7 @@ def seed() -> tuple[list[Seed], list[tuple[tuple[str, str], tuple[str, str]]]]: # Asia Pacific holds no country: a role placed there alone keeps the # host's country, and one beside a country is that country. add("continent", "asia-pacific", "Asia Pacific", None, ("asia pacific", "apac")) + add(*ANYWHERE, "Anywhere", None, ()) for code, names in _COUNTRY_LABELS.items(): add("country", code, names[0], europe if code in _EUROPE else None, _COUNTRY_FOLDS[code]) for country, code, name in _subdivisions(): @@ -892,6 +1026,9 @@ class Gazetteer: self._link(whole, part) self._ancestors: dict[int, frozenset[int]] = {} self._resolved: dict[str, tuple[int, ...]] = {} + self._overlap: dict[frozenset[int], frozenset[int]] = {} + self._names: dict[int, tuple[str, ...]] | None = None + self.anywhere: int | None = self.by_key.get(ANYWHERE) def _link(self, whole: int, part: int) -> None: self._up.setdefault(part, set()).add(whole) @@ -921,6 +1058,85 @@ class Gazetteer: todo += self._down.get(place_id, ()) return frozenset(seen) + def overlap(self, ids: Iterable[int]) -> frozenset[int]: + """Every place a remote area may hold to overlap these: the places, + what lies under them, every place one of those lies in, and Anywhere. + Denmark's holds Europe, and Øresund through the Capital Region, so a + role remote within Europe is open to someone in Denmark.""" + key = frozenset(ids) + if (found := self._overlap.get(key)) is None: + under = self.under(key) + wider = set(under) + for place_id in under: + wider |= self.ancestors(place_id) + if self.anywhere is not None: + wider.add(self.anywhere) + found = self._overlap[key] = frozenset(wider) + return found + + def names(self, place_id: int) -> tuple[str, ...]: + """The folds a place is written as: its aliases and its name.""" + if self._names is None: + by_id: dict[int, dict[str, None]] = {} + for folded, found in self.aliases.items(): + for one in found: + by_id.setdefault(one, {})[folded] = None + for place in self.places.values(): + by_id.setdefault(place.id, {})[fold(place.name)] = None + self._names = {place_id: tuple(folds) for place_id, folds in by_id.items()} + return self._names.get(place_id, ()) + + def token(self, place_id: int) -> str: + """The filter token that names a place: `country:DK`, `country:EU` + for Europe, `region:skane` for a region of `REGIONS`, `anywhere` for + Anywhere, else `place:` and the name folded.""" + place = self.places[place_id] + if place_id == self.anywhere: + return "anywhere" + if (place.kind, place.key) == ("continent", "europe"): + return "country:EU" + if place.kind == "country": + return f"country:{place.key}" + if place.kind == "region" and place.key in REGIONS: + return f"region:{place.key}" + return f"place:{fold(place.name)}" + + def remote_places(self, text: str, marked: bool = False) -> tuple[int, ...]: + """The places the parts of a place text that say remote + (`remote_part`) name, where a remote hire may live: `Germany + (Remote)` is Germany, `Cardiff, London or Remote (UK)` the United + Kingdom, `Home Based - Worldwide` and `Remote, Global` Anywhere. + `marked` takes every part, for a name the model read as where a + remote hire may live ("Europe", "anywhere"). + `global` and `international` are Anywhere only here, so `Global HQ, + Berlin` stays Berlin; beside another place Anywhere is dropped, so + `Anywhere in France, Belgium` is the two countries. `Remote` alone + names none.""" + found: list[int] = [] + for part in _PART.split(_AND_HOME.sub("", text or "")): + if not (marked or remote_part(part)): + continue + ids = [p for p in self.resolve(part) if p != self.anywhere] + if self.anywhere is not None and any( + f" {word} " in f" {fold(part)} " for word in _ANYWHERE_IF_REMOTE): + ids.append(self.anywhere) + if len(ids) > 1 and self.anywhere in ids: + ids.remove(self.anywhere) + found += ids + return tuple(dict.fromkeys(found)) + + def area_label(self, ids: Sequence[int]) -> str: + """A remote area as a row shows it: the names in order, Anywhere as + "anywhere", "United States, Canada". More than three places are + shown as the countries they lie in, so twenty German states read + "Germany".""" + found = [p for p in ids if p in self.places] + if len(found) > 3: + found = list(dict.fromkeys( + self.by_key.get(("country", self.country(p)), p) for p in found)) + return ", ".join("anywhere" if p == self.anywhere else self.places[p].name + for p in found) + def country(self, place_id: int) -> str: """The ISO code of the country a place is in, '' above or outside one.""" place: Place | None = self.places.get(place_id) @@ -1014,9 +1230,15 @@ class Gazetteer: """The places a filter token names, before what lies in them: `country:DK` is Denmark and `country:EU` Europe, `region:skane` Skåne, `place:malmo` every place written so (a town, or a region or a country by name). - None when the gazetteer knows no such place.""" + A `remote:` token names the places of its place token, `remote:anywhere` + Anywhere, `remote:any` none. None when the gazetteer knows no such + place.""" tok = parse(token) - if tok.kind == "country": + if tok.kind == "remote": + if tok.value != "anywhere": + return self.lookup(tok.value) if tok.value not in ("", "any") else None + found = {self.anywhere} if self.anywhere is not None else set() + elif tok.kind == "country": key = ("continent", "europe") if tok.value == "EU" else ("country", tok.value) found = {self.by_key[key]} if key in self.by_key else set() elif tok.kind == "region": diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 24995a9..1f95c81 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -50,7 +50,7 @@ def names(text: str) -> list[str]: def test_the_seed_holds_the_dictionaries_in_code() -> None: seeds, includes = seed() kinds = Counter(s.kind for s in seeds) - assert {s.key for s in seeds if s.kind == "continent"} == {"europe", "asia-pacific"} + assert {s.key for s in seeds if s.kind == "continent"} == {"europe", "asia-pacific", "world"} assert kinds["country"] == sum(1 for line in _COUNTRY_NAMES.split("\n") if line) keys = {(s.kind, s.key) for s in seeds} assert {("region", key) for key in places.REGIONS} <= keys