Compose remote as model, host word, place line and map a remote role's area in prep
The area is the model's names, else the parts of the place texts that say remote, else the resolved places as their countries. VERSION 3 maps every open role again. run_roles maps the roles extract answered. "Remote Sensing" in a title is no remote work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
57cac5f896
commit
e4c0de4465
5 files changed
+139
-36
No files matched your search
@@ -7,16 +7,18 @@ and embedding. One door: `from app.prep import ...`.
|
||||
words.py seniority, employment and remote: the host's table, the
|
||||
vocabulary itself, the shared remote words
|
||||
country.py the country from the resolved places, else the host's
|
||||
area.py a remote role's area: where a remote hire may live
|
||||
hosts/ one table file per reader (`hosts.TABLES`)
|
||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold;
|
||||
`run_roles`: a few roles at once, for `extract`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.prep.country import country
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run, run_roles
|
||||
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||
|
||||
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
||||
"employment", "fingerprint", "lookup", "remote", "run", "run_roles", "seniority"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""A remote role's area: the places a remote hire may live in.
|
||||
|
||||
`area` takes the first of these that names a place:
|
||||
|
||||
1. the model's `remote_where`, the names `extract` read in the posting,
|
||||
each read as a place text that says remote ("Europe", "anywhere")
|
||||
2. the parts of the role's place texts that say remote in words
|
||||
(`Gazetteer.remote_places`): "Germany (Remote)", "Home Based - EMEA",
|
||||
"Remote, Global"
|
||||
3. the role's resolved places, each as its country: Databricks' "Denmark"
|
||||
with the host's remote flag is Denmark, and Airtame's "København,
|
||||
Hovedstaden, Denmark; Remote" too, since an office town says where the
|
||||
company is, not that the hire must live in it. A place the reader
|
||||
marked remote without the word is the host's flag, and read so.
|
||||
|
||||
"Remote" alone names no place, so a role with nothing else has no area: it
|
||||
is remote, area not stated. The area is kept as its widest places
|
||||
(`widest`), in the order found.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
from app import places
|
||||
|
||||
|
||||
def widest(gaz: places.Gazetteer, ids: Iterable[int]) -> tuple[int, ...]:
|
||||
"""The places no other one of them lies in, in order, each once:
|
||||
Germany and Europe is Europe. Anywhere alone when it is one of them,
|
||||
since it holds every other."""
|
||||
found = tuple(dict.fromkeys(ids))
|
||||
if gaz.anywhere in found:
|
||||
return (gaz.anywhere,) if gaz.anywhere is not None else ()
|
||||
return tuple(p for p in found if not any(q in found for q in gaz.ancestors(p)))
|
||||
|
||||
|
||||
def _country(gaz: places.Gazetteer, place_id: int) -> int:
|
||||
"""The country a place lies in, else the place itself (a continent,
|
||||
Øresund)."""
|
||||
return gaz.by_key.get(("country", gaz.country(place_id)), place_id)
|
||||
|
||||
|
||||
def area(gaz: places.Gazetteer, texts: Sequence[str], place_ids: Sequence[int],
|
||||
model: Sequence[str] | None = None) -> tuple[int, ...]:
|
||||
"""The area of a remote role. `texts` are its place texts, `place_ids`
|
||||
its resolved places, `model` the names `extract` read (None never asked,
|
||||
`[]` not stated)."""
|
||||
for found in (
|
||||
[p for name in model or () for p in gaz.remote_places(name, marked=True)],
|
||||
[p for text in texts for p in gaz.remote_places(text)],
|
||||
[_country(gaz, p) for p in place_ids]):
|
||||
if found:
|
||||
return widest(gaz, found)
|
||||
return ()
|
||||
+53
-16
@@ -3,14 +3,17 @@
|
||||
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||
and marks the role due. `run` maps every due role: its company through
|
||||
`company_aliases`, its places against the gazetteer, its level, contract
|
||||
and remote through the host's tables (`app.prep.words`), and its country
|
||||
from the places (`app.prep.country`). When the tables or the gazetteer in
|
||||
code (`places.seed`) changed since the last run (`fingerprint`, kept in
|
||||
state as `prep:tables`), the new places and spellings are added to the
|
||||
database and every open role is mapped again from its stored words, with
|
||||
no new pull. Then the words no
|
||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||
copies across the whole database.
|
||||
and remote through the host's tables (`app.prep.words`), its country from
|
||||
the places (`app.prep.country`), and a remote role's area
|
||||
(`app.prep.area`). `remote` is the model's (`remote_model`), else the
|
||||
host's word, else what the place line or a search role's title says. When
|
||||
the tables or the gazetteer in code (`places.seed`) changed since the last
|
||||
run (`fingerprint`, kept in state as `prep:tables`), the new places and
|
||||
spellings are added to the database and every open role is mapped again
|
||||
from its stored words, with no new pull. Then the words no table knows are
|
||||
listed in `unmapped`, and `Store.rebuild_dupes` folds the copies across the
|
||||
whole database. `run_roles` maps a few roles outside a run, for `extract`,
|
||||
whose model answer changes remote and area.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,14 +27,16 @@ from app import places
|
||||
from app.models import company_key
|
||||
from app.models import company_fold
|
||||
from app.prep import words
|
||||
from app.prep.area import area
|
||||
from app.prep.country import country, of_host
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.words import Tables
|
||||
from app.storage import Store
|
||||
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes.
|
||||
VERSION = 2
|
||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||
#: remote area, and remote as the model's before the host's word.
|
||||
VERSION = 3
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
@@ -75,11 +80,30 @@ def _places(raw: Any) -> list[Any]:
|
||||
return found if isinstance(found, list) else []
|
||||
|
||||
|
||||
def _texts(row: Mapping[str, Any]) -> list[str]:
|
||||
"""The role's place texts: `raw`'s places, else the line cut at `;`."""
|
||||
given = _places(row["raw_places"])
|
||||
return [words.place_text(p) for p in given] if given \
|
||||
else str(row["place"] or "").split(";")
|
||||
|
||||
|
||||
def _model_names(raw: Any) -> list[str] | None:
|
||||
"""`remote_where_model` read: None never asked, else the names."""
|
||||
if raw is None:
|
||||
return None
|
||||
return [str(name) for name in _places(raw)]
|
||||
|
||||
|
||||
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||
host = str(row["host"] or "")
|
||||
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||
texts = _texts(row)
|
||||
remote = row["remote_model"] \
|
||||
or words.remote(host, row["remote_host"] or None, (), "", tables) \
|
||||
or words.remote(host, None, _places(row["raw_places"]), row["place"], tables,
|
||||
title=row["title"] if row["source"] == "search" else "")
|
||||
return {
|
||||
"uid": row["uid"],
|
||||
"due": row["prep_due"],
|
||||
@@ -87,10 +111,9 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
||||
"seniority_stated": stated,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
||||
"remote_places": words.remote(
|
||||
host, None, _places(row["raw_places"]), row["place"], tables,
|
||||
title=row["title"] if row["source"] == "search" else ""),
|
||||
"remote": remote,
|
||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||
if remote == "remote" else ()),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
}
|
||||
|
||||
@@ -98,8 +121,8 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
||||
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or (found["remote"] and found["remote"] != row["remote"]) \
|
||||
or (found["remote_places"] and not row["remote"]) \
|
||||
or found["remote"] != row["remote"] \
|
||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||
"seniority_title", "employment"))
|
||||
@@ -143,3 +166,17 @@ def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||
copies = store.rebuild_dupes().copies if dupes else None
|
||||
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||
copies=copies)
|
||||
|
||||
|
||||
def run_roles(store: Store, uids: Sequence[str], tables: Tables = TABLES) -> int:
|
||||
"""Map these roles now, due or not, and nothing else: what `extract`
|
||||
calls for the roles it answered, so the rescore that follows sees the
|
||||
model's remote and area. Returns the rows written."""
|
||||
if not uids:
|
||||
return 0
|
||||
rows = store.prep_rows("all", uids=uids)
|
||||
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||
gaz = store.gazetteer()
|
||||
folds = store.company_folds()
|
||||
return store.write_prep([map_role(r, gaz, folds, ids.get(r["uid"], ()), tables)
|
||||
for r in rows])
|
||||
+12
-5
@@ -52,6 +52,12 @@ _REMOTE_LINE = re.compile(
|
||||
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
|
||||
|
||||
#: "Remote" in a title that names a field or a product, not how the role is
|
||||
#: worked: "Remote Sensing Specialist", "Remote Control Engineer".
|
||||
_NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\b",
|
||||
re.IGNORECASE)
|
||||
|
||||
|
||||
def _line_word(text: str) -> str:
|
||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||
if found := remote_of(text):
|
||||
@@ -98,7 +104,8 @@ def employment(host: str, word: str,
|
||||
return lookup("employment", host, word, tables) or ""
|
||||
|
||||
|
||||
def _place_text(place: Any) -> str:
|
||||
def place_text(place: Any) -> str:
|
||||
"""A place of `raw`'s `places`, a dict with `text` or a string, as its text."""
|
||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||
|
||||
|
||||
@@ -114,17 +121,17 @@ def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||
nothing, the place line or the place texts are read for a word
|
||||
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
|
||||
`title` when given, which the stage gives for a search provider's role
|
||||
only ("Backend Developer - Remote"), and last a place the reader marked
|
||||
remote is remote."""
|
||||
only ("Backend Developer - Remote", but not "Remote Sensing Analyst"),
|
||||
and last a place the reader marked remote is remote."""
|
||||
if value is True or value == "true":
|
||||
return "remote"
|
||||
said = value not in (None, False, "false")
|
||||
if said and (found := lookup("remote", host, str(value), tables)):
|
||||
return found
|
||||
places = list(places)
|
||||
if found := _line_word(line or "; ".join(map(_place_text, places))):
|
||||
if found := _line_word(line or "; ".join(map(place_text, places))):
|
||||
return found
|
||||
if title and (found := _line_word(title)):
|
||||
if title and (found := _line_word(_NOT_REMOTE_WORK.sub(" ", title))):
|
||||
return found
|
||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||
return "remote"
|
||||
|
||||
+14
-12
@@ -205,21 +205,23 @@ def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
||||
assert row(store, "1")["prep_due"] == 2
|
||||
|
||||
|
||||
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
|
||||
def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
||||
store: Store) -> None:
|
||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
||||
uid = row(store, "1")["uid"]
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
|
||||
"WHERE uid = ?", (uid,))
|
||||
prep.run(store, tables=TABLES)
|
||||
mapped = row(store, "1")
|
||||
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
||||
("senior", "lead", "remote")
|
||||
# The host's own statement does overwrite.
|
||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
|
||||
prep.run(store, tables=TABLES)
|
||||
assert row(store, "1")["remote"] == "hybrid"
|
||||
# The host's own word wins over the line.
|
||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="fully"))
|
||||
prep.run(store, tables=TABLES)
|
||||
assert row(store, "1")["remote"] == "remote"
|
||||
uid = row(store, "1")["uid"]
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = 'lead', remote_model = 'onsite' "
|
||||
"WHERE uid = ?", (uid,))
|
||||
prep.run(store, tables=TABLES, remap=True)
|
||||
mapped = row(store, "1")
|
||||
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
||||
("senior", "lead", "onsite")
|
||||
|
||||
|
||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
||||
@@ -314,7 +316,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26]
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
|
||||
Reference in new issue
Block a user