Mark a role whose title or text says remote while no field does, in prep with no model, and let it meet a gate on remote until extract has read it (migration 35, roles.remote_text)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-06 14:02:46 +02:00
1 parent f141cffc3c
commit 49929372c7
14 files changed
+144 -36

No files matched your search

+6 -2
View File
@@ -37,8 +37,9 @@ from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes. 3: the
#: remote area, and remote as the model's before the host's word. 4: the
#: deadline, the host's before the model's.
VERSION = 4
#: deadline, the host's before the model's. 5: `remote_text`, the title or
#: text saying remote while no field does.
VERSION = 5
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words, the gazetteer the
code describes and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
"places": repr(places.seed())}, sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": remote,
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
if remote == "remote" else ()),
"country": country(gaz, place_ids, row["country_host"]),
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or found["remote"] != row["remote"] \
or found["remote_text"] != row["remote_text"] \
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
or (found["country"] and found["country"] != row["country"]) \
or found["deadline"] != row["deadline"] \
+33
View File
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
re.IGNORECASE)
#: Phrases in a title or text that say the role is worked remotely, and the
#: phrases that take one back. Both are read on the lowercased text with its
#: hyphens and runs of spaces squashed to one space. A match marks the role as
#: possibly remote (`remote_in_text`); it decides nothing by itself.
REMOTE_TEXT: tuple[str, ...] = (
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
"remote working", "work remotely", "working remotely", "remote position", "remote role",
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
)
REMOTE_TEXT_NOT: tuple[str, ...] = (
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
"remote control", "remote monitoring",
)
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
_SQUASHED = re.compile(r"[\s-]+")
def remote_in_text(title: str, text: str) -> bool:
"""Whether the title or the text says the role is worked remotely: a
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
out. Cheap evidence, no model: a role marked so passes a location gate
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
false mark costs a cut slot at most, and a missed one nothing new."""
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
def _line_word(text: str) -> str:
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
if found := remote_of(text):
+15
View File
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
-- 1 when the title or text says the role is worked remotely while no
-- field does (prep, app.prep.words.remote_in_text): the location gate
-- lets such a role through until extract has read it.
remote_text INTEGER NOT NULL DEFAULT 0,
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
@@ -760,6 +764,8 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# judge reads every role again (judge.FORMAT).
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
"location gate", lambda conn: _add_remote_text(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -1009,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
def _add_remote_text(conn: sqlite3.Connection) -> None:
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
which maps every open role again since its rules changed. Runs twice
without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
if "remote_text" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
def _add_remote_area(conn: sqlite3.Connection) -> None:
"""Migration 27's step: `roles.remote_model` empty and
`roles.remote_where_model` NULL for every role, so `extract` reads the
+12 -4
View File
@@ -355,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
return ("r.languages IN ('', '[]')",
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
f"WHERE LOWER(value) NOT IN ({marks}))", values)
if gate.field == "remote" and "remote" in values:
# A role whose text says remote while no field does is possibly
# remote (`remote_text`, prep): it meets a gate on remote until
# extract has read it, and is not empty for it.
return (f"({column} = '' AND r.remote_text = 0)",
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
@@ -1625,8 +1631,9 @@ class Store:
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country, remote_model, remote_where_model,
deadline_host, deadline_model, deadline,
employment, remote, remote_text, country, remote_model,
remote_where_model, deadline_host, deadline_model, deadline,
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
@@ -1652,8 +1659,8 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
@@ -1669,6 +1676,7 @@ class Store:
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = :remote,
remote_text = :remote_text,
country = COALESCE(NULLIF(:country, ''), country),
deadline = :deadline,
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END