Compose remote as model, host word, place line and map a remote role's area in prep
The area is the model's names, else the parts of the place texts that say remote, else the resolved places as their countries. VERSION 3 maps every open role again. run_roles maps the roles extract answered. "Remote Sensing" in a title is no remote work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
57cac5f896
commit
e4c0de4465
5 files changed
+139
-36
No files matched your search
@@ -7,16 +7,18 @@ and embedding. One door: `from app.prep import ...`.
|
|||||||
words.py seniority, employment and remote: the host's table, the
|
words.py seniority, employment and remote: the host's table, the
|
||||||
vocabulary itself, the shared remote words
|
vocabulary itself, the shared remote words
|
||||||
country.py the country from the resolved places, else the host's
|
country.py the country from the resolved places, else the host's
|
||||||
|
area.py a remote role's area: where a remote hire may live
|
||||||
hosts/ one table file per reader (`hosts.TABLES`)
|
hosts/ one table file per reader (`hosts.TABLES`)
|
||||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
stage.py `run`: the stage over the due roles, `unmapped` and the fold;
|
||||||
|
`run_roles`: a few roles at once, for `extract`
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.prep.country import country
|
from app.prep.country import country
|
||||||
from app.prep.hosts import TABLES
|
from app.prep.hosts import TABLES
|
||||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
from app.prep.stage import STATE, Prepped, company, fingerprint, run, run_roles
|
||||||
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||||
|
|
||||||
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||||
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
"employment", "fingerprint", "lookup", "remote", "run", "run_roles", "seniority"]
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""A remote role's area: the places a remote hire may live in.
|
||||||
|
|
||||||
|
`area` takes the first of these that names a place:
|
||||||
|
|
||||||
|
1. the model's `remote_where`, the names `extract` read in the posting,
|
||||||
|
each read as a place text that says remote ("Europe", "anywhere")
|
||||||
|
2. the parts of the role's place texts that say remote in words
|
||||||
|
(`Gazetteer.remote_places`): "Germany (Remote)", "Home Based - EMEA",
|
||||||
|
"Remote, Global"
|
||||||
|
3. the role's resolved places, each as its country: Databricks' "Denmark"
|
||||||
|
with the host's remote flag is Denmark, and Airtame's "København,
|
||||||
|
Hovedstaden, Denmark; Remote" too, since an office town says where the
|
||||||
|
company is, not that the hire must live in it. A place the reader
|
||||||
|
marked remote without the word is the host's flag, and read so.
|
||||||
|
|
||||||
|
"Remote" alone names no place, so a role with nothing else has no area: it
|
||||||
|
is remote, area not stated. The area is kept as its widest places
|
||||||
|
(`widest`), in the order found.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterable, Sequence
|
||||||
|
|
||||||
|
from app import places
|
||||||
|
|
||||||
|
|
||||||
|
def widest(gaz: places.Gazetteer, ids: Iterable[int]) -> tuple[int, ...]:
|
||||||
|
"""The places no other one of them lies in, in order, each once:
|
||||||
|
Germany and Europe is Europe. Anywhere alone when it is one of them,
|
||||||
|
since it holds every other."""
|
||||||
|
found = tuple(dict.fromkeys(ids))
|
||||||
|
if gaz.anywhere in found:
|
||||||
|
return (gaz.anywhere,) if gaz.anywhere is not None else ()
|
||||||
|
return tuple(p for p in found if not any(q in found for q in gaz.ancestors(p)))
|
||||||
|
|
||||||
|
|
||||||
|
def _country(gaz: places.Gazetteer, place_id: int) -> int:
|
||||||
|
"""The country a place lies in, else the place itself (a continent,
|
||||||
|
Øresund)."""
|
||||||
|
return gaz.by_key.get(("country", gaz.country(place_id)), place_id)
|
||||||
|
|
||||||
|
|
||||||
|
def area(gaz: places.Gazetteer, texts: Sequence[str], place_ids: Sequence[int],
|
||||||
|
model: Sequence[str] | None = None) -> tuple[int, ...]:
|
||||||
|
"""The area of a remote role. `texts` are its place texts, `place_ids`
|
||||||
|
its resolved places, `model` the names `extract` read (None never asked,
|
||||||
|
`[]` not stated)."""
|
||||||
|
for found in (
|
||||||
|
[p for name in model or () for p in gaz.remote_places(name, marked=True)],
|
||||||
|
[p for text in texts for p in gaz.remote_places(text)],
|
||||||
|
[_country(gaz, p) for p in place_ids]):
|
||||||
|
if found:
|
||||||
|
return widest(gaz, found)
|
||||||
|
return ()
|
||||||
+53
-16
@@ -3,14 +3,17 @@
|
|||||||
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||||
and marks the role due. `run` maps every due role: its company through
|
and marks the role due. `run` maps every due role: its company through
|
||||||
`company_aliases`, its places against the gazetteer, its level, contract
|
`company_aliases`, its places against the gazetteer, its level, contract
|
||||||
and remote through the host's tables (`app.prep.words`), and its country
|
and remote through the host's tables (`app.prep.words`), its country from
|
||||||
from the places (`app.prep.country`). When the tables or the gazetteer in
|
the places (`app.prep.country`), and a remote role's area
|
||||||
code (`places.seed`) changed since the last run (`fingerprint`, kept in
|
(`app.prep.area`). `remote` is the model's (`remote_model`), else the
|
||||||
state as `prep:tables`), the new places and spellings are added to the
|
host's word, else what the place line or a search role's title says. When
|
||||||
database and every open role is mapped again from its stored words, with
|
the tables or the gazetteer in code (`places.seed`) changed since the last
|
||||||
no new pull. Then the words no
|
run (`fingerprint`, kept in state as `prep:tables`), the new places and
|
||||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
spellings are added to the database and every open role is mapped again
|
||||||
copies across the whole database.
|
from its stored words, with no new pull. Then the words no table knows are
|
||||||
|
listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the
|
||||||
|
whole database. `run_roles` maps a few roles outside a run, for `extract`,
|
||||||
|
whose model answer changes remote and area.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -24,14 +27,16 @@ from app import places
|
|||||||
from app.models import company_key
|
from app.models import company_key
|
||||||
from app.models import company_fold
|
from app.models import company_fold
|
||||||
from app.prep import words
|
from app.prep import words
|
||||||
|
from app.prep.area import area
|
||||||
from app.prep.country import country, of_host
|
from app.prep.country import country, of_host
|
||||||
from app.prep.hosts import TABLES
|
from app.prep.hosts import TABLES
|
||||||
from app.prep.words import Tables
|
from app.prep.words import Tables
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
|
||||||
#: Raised when the rules in this package change the way a stored word maps,
|
#: Raised when the rules in this package change the way a stored word maps,
|
||||||
#: so every open role is mapped again as it is after a table changes.
|
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||||
VERSION = 2
|
#: remote area, and remote as the model's before the host's word.
|
||||||
|
VERSION = 3
|
||||||
|
|
||||||
#: The state key the last run's `fingerprint` is kept under.
|
#: The state key the last run's `fingerprint` is kept under.
|
||||||
STATE = "prep:tables"
|
STATE = "prep:tables"
|
||||||
@@ -75,11 +80,30 @@ def _places(raw: Any) -> list[Any]:
|
|||||||
return found if isinstance(found, list) else []
|
return found if isinstance(found, list) else []
|
||||||
|
|
||||||
|
|
||||||
|
def _texts(row: Mapping[str, Any]) -> list[str]:
|
||||||
|
"""The role's place texts: `raw`'s places, else the line cut at `;`."""
|
||||||
|
given = _places(row["raw_places"])
|
||||||
|
return [words.place_text(p) for p in given] if given \
|
||||||
|
else str(row["place"] or "").split(";")
|
||||||
|
|
||||||
|
|
||||||
|
def _model_names(raw: Any) -> list[str] | None:
|
||||||
|
"""`remote_where_model` read: None never asked, else the names."""
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
return [str(name) for name in _places(raw)]
|
||||||
|
|
||||||
|
|
||||||
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||||
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||||
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||||
host = str(row["host"] or "")
|
host = str(row["host"] or "")
|
||||||
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||||
|
texts = _texts(row)
|
||||||
|
remote = row["remote_model"] \
|
||||||
|
or words.remote(host, row["remote_host"] or None, (), "", tables) \
|
||||||
|
or words.remote(host, None, _places(row["raw_places"]), row["place"], tables,
|
||||||
|
title=row["title"] if row["source"] == "search" else "")
|
||||||
return {
|
return {
|
||||||
"uid": row["uid"],
|
"uid": row["uid"],
|
||||||
"due": row["prep_due"],
|
"due": row["prep_due"],
|
||||||
@@ -87,10 +111,9 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
|||||||
"seniority_stated": stated,
|
"seniority_stated": stated,
|
||||||
"seniority_title": titled,
|
"seniority_title": titled,
|
||||||
"employment": words.employment(host, row["employment_host"], tables),
|
"employment": words.employment(host, row["employment_host"], tables),
|
||||||
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
"remote": remote,
|
||||||
"remote_places": words.remote(
|
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||||
host, None, _places(row["raw_places"]), row["place"], tables,
|
if remote == "remote" else ()),
|
||||||
title=row["title"] if row["source"] == "search" else ""),
|
|
||||||
"country": country(gaz, place_ids, row["country_host"]),
|
"country": country(gaz, place_ids, row["country_host"]),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,8 +121,8 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
|||||||
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||||
or (found["remote"] and found["remote"] != row["remote"]) \
|
or found["remote"] != row["remote"] \
|
||||||
or (found["remote_places"] and not row["remote"]) \
|
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||||
or (found["country"] and found["country"] != row["country"]) \
|
or (found["country"] and found["country"] != row["country"]) \
|
||||||
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||||
"seniority_title", "employment"))
|
"seniority_title", "employment"))
|
||||||
@@ -143,3 +166,17 @@ def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
|||||||
copies = store.rebuild_dupes().copies if dupes else None
|
copies = store.rebuild_dupes().copies if dupes else None
|
||||||
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||||
copies=copies)
|
copies=copies)
|
||||||
|
|
||||||
|
|
||||||
|
def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int:
|
||||||
|
"""Map these roles now, due or not, and nothing else: what `extract`
|
||||||
|
calls for the roles it answered, so the rescore that follows sees the
|
||||||
|
model's remote and area. Returns the rows written."""
|
||||||
|
if not uids:
|
||||||
|
return 0
|
||||||
|
rows = store.prep_rows("all", uids=uids)
|
||||||
|
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||||
|
gaz = store.gazetteer()
|
||||||
|
folds = store.company_folds()
|
||||||
|
return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables)
|
||||||
|
for r in rows])
|
||||||
+12
-5
@@ -52,6 +52,12 @@ _REMOTE_LINE = re.compile(
|
|||||||
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
#: "Remote" in a title that names a field or a product, not how the role is
|
||||||
|
#: worked: "Remote Sensing Specialist", "Remote Control Engineer".
|
||||||
|
_NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\b",
|
||||||
|
re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
def _line_word(text: str) -> str:
|
def _line_word(text: str) -> str:
|
||||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||||
if found := remote_of(text):
|
if found := remote_of(text):
|
||||||
@@ -98,7 +104,8 @@ def employment(host: str, word: str,
|
|||||||
return lookup("employment", host, word, tables) or ""
|
return lookup("employment", host, word, tables) or ""
|
||||||
|
|
||||||
|
|
||||||
def _place_text(place: Any) -> str:
|
def place_text(place: Any) -> str:
|
||||||
|
"""A place of `raw`'s `places`, a dict with `text` or a string, as its text."""
|
||||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||||
|
|
||||||
|
|
||||||
@@ -114,17 +121,17 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
|||||||
nothing, the place line or the place texts are read for a word
|
nothing, the place line or the place texts are read for a word
|
||||||
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
|
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
|
||||||
`title` when given, which the stage gives for a search provider's role
|
`title` when given, which the stage gives for a search provider's role
|
||||||
only ("Backend Developer - Remote"), and last a place the reader marked
|
only ("Backend Developer - Remote", but not "Remote Sensing Analyst"),
|
||||||
remote is remote."""
|
and last a place the reader marked remote is remote."""
|
||||||
if value is True or value == "true":
|
if value is True or value == "true":
|
||||||
return "remote"
|
return "remote"
|
||||||
said = value not in (None, False, "false")
|
said = value not in (None, False, "false")
|
||||||
if said and (found := lookup("remote", host, str(value), tables)):
|
if said and (found := lookup("remote", host, str(value), tables)):
|
||||||
return found
|
return found
|
||||||
places = list(places)
|
places = list(places)
|
||||||
if found := _line_word(line or "; ".join(map(_place_text, places))):
|
if found := _line_word(line or "; ".join(map(place_text, places))):
|
||||||
return found
|
return found
|
||||||
if title and (found := _line_word(title)):
|
if title and (found := _line_word(_NOT_REMOTE_WORK.sub(" ", title))):
|
||||||
return found
|
return found
|
||||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||||
return "remote"
|
return "remote"
|
||||||
|
|||||||
+14
-12
@@ -205,21 +205,23 @@ def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
|||||||
assert row(store, "1")["prep_due"] == 2
|
assert row(store, "1")["prep_due"] == 2
|
||||||
|
|
||||||
|
|
||||||
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
|
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
||||||
uid = row(store, "1")["uid"]
|
|
||||||
with store.transaction() as conn:
|
|
||||||
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
|
|
||||||
"WHERE uid = ?", (uid,))
|
|
||||||
prep.run(store, tables=TABLES)
|
|
||||||
mapped = row(store, "1")
|
|
||||||
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
|
||||||
("senior", "lead", "remote")
|
|
||||||
# The host's own statement does overwrite.
|
|
||||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
|
|
||||||
prep.run(store, tables=TABLES)
|
prep.run(store, tables=TABLES)
|
||||||
assert row(store, "1")["remote"] == "hybrid"
|
assert row(store, "1")["remote"] == "hybrid"
|
||||||
|
# The host's own word wins over the line.
|
||||||
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
assert row(store, "1")["remote"] == "remote"
|
||||||
|
uid = row(store, "1")["uid"]
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
|
||||||
|
"WHERE uid = ?", (uid,))
|
||||||
|
prep.run(store, tables=TABLES, remap=True)
|
||||||
|
mapped = row(store, "1")
|
||||||
|
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
||||||
|
("senior", "lead", "onsite")
|
||||||
|
|
||||||
|
|
||||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
||||||
@@ -314,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26]
|
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27]
|
||||||
one = row(migrated, "1")
|
one = row(migrated, "1")
|
||||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||||
|
|||||||
Reference in new issue
Block a user