diff --git a/app/prep/__init__.py b/app/prep/__init__.py index 7d64ac8..ff3aacb 100644 --- a/app/prep/__init__.py +++ b/app/prep/__init__.py @@ -7,16 +7,18 @@ and embedding. One door: `from app.prep import ...`. words.py seniority, employment and remote: the host's table, the vocabulary itself, the shared remote words country.py the country from the resolved places, else the host's + area.py a remote role's area: where a remote hire may live hosts/ one table file per reader (`hosts.TABLES`) - stage.py `run`: the stage over the due roles, `unmapped` and the fold + stage.py `run`: the stage over the due roles, `unmapped` and the fold; + `run_roles`: a few roles at once, for `extract` """ from __future__ import annotations from app.prep.country import country from app.prep.hosts import TABLES -from app.prep.stage import STATE, Prepped, company, fingerprint, run +from app.prep.stage import STATE, Prepped, company, fingerprint, run, run_roles from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority __all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country", - "employment", "fingerprint", "lookup", "remote", "run", "seniority"] + "employment", "fingerprint", "lookup", "remote", "run", "run_roles", "seniority"] diff --git a/app/prep/area.py b/app/prep/area.py new file mode 100644 index 0000000..f78c1ba --- /dev/null +++ b/app/prep/area.py @@ -0,0 +1,55 @@ +"""A remote role's area: the places a remote hire may live in. + +`area` takes the first of these that names a place: + +1. the model's `remote_where`, the names `extract` read in the posting, + each read as a place text that says remote ("Europe", "anywhere") +2. the parts of the role's place texts that say remote in words + (`Gazetteer.remote_places`): "Germany (Remote)", "Home Based - EMEA", + "Remote, Global" +3. the role's resolved places, each as its country: Databricks' "Denmark" + with the host's remote flag is Denmark, and Airtame's "København, + Hovedstaden, Denmark; Remote" too, since an office town says where the + company is, not that the hire must live in it. A place the reader + marked remote without the word is the host's flag, and read so. + +"Remote" alone names no place, so a role with nothing else has no area: it +is remote, area not stated. The area is kept as its widest places +(`widest`), in the order found. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence + +from app import places + + +def widest(gaz: places.Gazetteer, ids: Iterable[int]) -> tuple[int, ...]: + """The places no other one of them lies in, in order, each once: + Germany and Europe is Europe. Anywhere alone when it is one of them, + since it holds every other.""" + found = tuple(dict.fromkeys(ids)) + if gaz.anywhere in found: + return (gaz.anywhere,) if gaz.anywhere is not None else () + return tuple(p for p in found if not any(q in found for q in gaz.ancestors(p))) + + +def _country(gaz: places.Gazetteer, place_id: int) -> int: + """The country a place lies in, else the place itself (a continent, + Øresund).""" + return gaz.by_key.get(("country", gaz.country(place_id)), place_id) + + +def area(gaz: places.Gazetteer, texts: Sequence[str], place_ids: Sequence[int], + model: Sequence[str] | None = None) -> tuple[int, ...]: + """The area of a remote role. `texts` are its place texts, `place_ids` + its resolved places, `model` the names `extract` read (None never asked, + `[]` not stated).""" + for found in ( + [p for name in model or () for p in gaz.remote_places(name, marked=True)], + [p for text in texts for p in gaz.remote_places(text)], + [_country(gaz, p) for p in place_ids]): + if found: + return widest(gaz, found) + return () diff --git a/app/prep/stage.py b/app/prep/stage.py index 0a6bb52..b27d6b9 100644 --- a/app/prep/stage.py +++ b/app/prep/stage.py @@ -3,14 +3,17 @@ A pull stores the host's own words and place texts (`Store.upsert_role`) and marks the role due. `run` maps every due role: its company through `company_aliases`, its places against the gazetteer, its level, contract -and remote through the host's tables (`app.prep.words`), and its country -from the places (`app.prep.country`). When the tables or the gazetteer in -code (`places.seed`) changed since the last run (`fingerprint`, kept in -state as `prep:tables`), the new places and spellings are added to the -database and every open role is mapped again from its stored words, with -no new pull. Then the words no -table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the -copies across the whole database. +and remote through the host's tables (`app.prep.words`), its country from +the places (`app.prep.country`), and a remote role's area +(`app.prep.area`). `remote` is the model's (`remote_model`), else the +host's word, else what the place line or a search role's title says. When +the tables or the gazetteer in code (`places.seed`) changed since the last +run (`fingerprint`, kept in state as `prep:tables`), the new places and +spellings are added to the database and every open role is mapped again +from its stored words, with no new pull. Then the words no table knows are +listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the +whole database. `run_roles` maps a few roles outside a run, for `extract`, +whose model answer changes remote and area. """ from __future__ import annotations @@ -24,14 +27,16 @@ from app import places from app.models import company_key from app.models import company_fold from app.prep import words +from app.prep.area import area from app.prep.country import country, of_host from app.prep.hosts import TABLES from app.prep.words import Tables from app.storage import Store #: Raised when the rules in this package change the way a stored word maps, -#: so every open role is mapped again as it is after a table changes. -VERSION = 2 +#: so every open role is mapped again as it is after a table changes. 3: the +#: remote area, and remote as the model's before the host's word. +VERSION = 3 #: The state key the last run's `fingerprint` is kept under. STATE = "prep:tables" @@ -75,11 +80,30 @@ def _places(raw: Any) -> list[Any]: return found if isinstance(found, list) else [] +def _texts(row: Mapping[str, Any]) -> list[str]: + """The role's place texts: `raw`'s places, else the line cut at `;`.""" + given = _places(row["raw_places"]) + return [words.place_text(p) for p in given] if given \ + else str(row["place"] or "").split(";") + + +def _model_names(raw: Any) -> list[str] | None: + """`remote_where_model` read: None never asked, else the names.""" + if raw is None: + return None + return [str(name) for name in _places(raw)] + + def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]], place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]: """One role as `Store.write_prep` takes it, from a `Store.prep_rows` row.""" host = str(row["host"] or "") stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables) + texts = _texts(row) + remote = row["remote_model"] \ + or words.remote(host, row["remote_host"] or None, (), "", tables) \ + or words.remote(host, None, _places(row["raw_places"]), row["place"], tables, + title=row["title"] if row["source"] == "search" else "") return { "uid": row["uid"], "due": row["prep_due"], @@ -87,10 +111,9 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, "seniority_stated": stated, "seniority_title": titled, "employment": words.employment(host, row["employment_host"], tables), - "remote": words.remote(host, row["remote_host"] or None, (), "", tables), - "remote_places": words.remote( - host, None, _places(row["raw_places"]), row["place"], tables, - title=row["title"] if row["source"] == "search" else ""), + "remote": remote, + "remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"])) + if remote == "remote" else ()), "country": country(gaz, place_ids, row["country_host"]), } @@ -98,8 +121,8 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool: level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"] return bool(row["prep_due"]) or level != row["seniority"] \ - or (found["remote"] and found["remote"] != row["remote"]) \ - or (found["remote_places"] and not row["remote"]) \ + or found["remote"] != row["remote"] \ + or found["remote_area"] != json.loads(row["remote_area"] or "[]") \ or (found["country"] and found["country"] != row["country"]) \ or any(found[c] != row[c] for c in ("company_key", "seniority_stated", "seniority_title", "employment")) @@ -143,3 +166,17 @@ def run(store: Store, *, remap: bool = False, tables: Tables = TABLES, copies = store.rebuild_dupes().copies if dupes else None return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left), copies=copies) + + +def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int: + """Map these roles now, due or not, and nothing else: what `extract` + calls for the roles it answered, so the rescore that follows sees the + model's remote and area. Returns the rows written.""" + if not uids: + return 0 + rows = store.prep_rows("all", uids=uids) + ids = store.resolve_role_places([r["uid"] for r in rows]) + gaz = store.gazetteer() + folds = store.company_folds() + return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables) + for r in rows]) diff --git a/app/prep/words.py b/app/prep/words.py index da1ee7b..e530042 100644 --- a/app/prep/words.py +++ b/app/prep/words.py @@ -52,6 +52,12 @@ _REMOTE_LINE = re.compile( _HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) +#: "Remote" in a title that names a field or a product, not how the role is +#: worked: "Remote Sensing Specialist", "Remote Control Engineer". +_NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\b", + re.IGNORECASE) + + def _line_word(text: str) -> str: """`remote_of`'s reading of a line, else its Danish and Swedish words.""" if found := remote_of(text): @@ -98,7 +104,8 @@ def employment(host: str, word: str, return lookup("employment", host, word, tables) or "" -def _place_text(place: Any) -> str: +def place_text(place: Any) -> str: + """A place of `raw`'s `places`, a dict with `text` or a string, as its text.""" return str((place.get("text") if isinstance(place, Mapping) else place) or "") @@ -114,17 +121,17 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "", nothing, the place line or the place texts are read for a word (`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then `title` when given, which the stage gives for a search provider's role - only ("Backend Developer - Remote"), and last a place the reader marked - remote is remote.""" + only ("Backend Developer - Remote", but not "Remote Sensing Analyst"), + and last a place the reader marked remote is remote.""" if value is True or value == "true": return "remote" said = value not in (None, False, "false") if said and (found := lookup("remote", host, str(value), tables)): return found places = list(places) - if found := _line_word(line or "; ".join(map(_place_text, places))): + if found := _line_word(line or "; ".join(map(place_text, places))): return found - if title and (found := _line_word(title)): + if title and (found := _line_word(_NOT_REMOTE_WORK.sub(" ", title))): return found if any(isinstance(p, Mapping) and p.get("remote") is True for p in places): return "remote" diff --git a/tests/test_prep.py b/tests/test_prep.py index 1f71411..28dca2b 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -205,21 +205,23 @@ def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None: assert row(store, "1")["prep_due"] == 2 -def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract( +def test_the_model_level_and_remote_win_over_the_host_and_the_place_line( store: Store) -> None: store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin")) - uid = row(store, "1")["uid"] - with store.transaction() as conn: - conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' " - "WHERE uid = ?", (uid,)) - prep.run(store, tables=TABLES) - mapped = row(store, "1") - assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \ - ("senior", "lead", "remote") - # The host's own statement does overwrite. - store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2")) prep.run(store, tables=TABLES) assert row(store, "1")["remote"] == "hybrid" + # The host's own word wins over the line. + store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully")) + prep.run(store, tables=TABLES) + assert row(store, "1")["remote"] == "remote" + uid = row(store, "1")["uid"] + with store.transaction() as conn: + conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' " + "WHERE uid = ?", (uid,)) + prep.run(store, tables=TABLES, remap=True) + mapped = row(store, "1") + assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \ + ("senior", "lead", "onsite") def test_the_company_resolves_through_its_aliases(store: Store) -> None: @@ -314,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [22, 23, 24, 25, 26] + assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27] one = row(migrated, "1") assert (one["seniority_host"], one["employment_host"], one["remote_host"], one["country_host"]) == ("senior", "full_time", "true", "DK")