Mark a role whose title or text says remote while no field does, in prep with no model, and let it meet a gate on remote until extract has read it (migration 35, roles.remote_text)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
f141cffc3c
commit
49929372c7
14 files changed
+144
-36
No files matched your search
+6
-2
@@ -37,8 +37,9 @@ from app.storage import Store
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||
#: remote area, and remote as the model's before the host's word. 4: the
|
||||
#: deadline, the host's before the model's.
|
||||
VERSION = 4
|
||||
#: deadline, the host's before the model's. 5: `remote_text`, the title or
|
||||
#: text saying remote while no field does.
|
||||
VERSION = 5
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||
code describes and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
|
||||
"places": repr(places.seed())}, sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": remote,
|
||||
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
|
||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||
if remote == "remote" else ()),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or found["remote"] != row["remote"] \
|
||||
or found["remote_text"] != row["remote_text"] \
|
||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or found["deadline"] != row["deadline"] \
|
||||
|
||||
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
|
||||
re.IGNORECASE)
|
||||
|
||||
|
||||
#: Phrases in a title or text that say the role is worked remotely, and the
|
||||
#: phrases that take one back. Both are read on the lowercased text with its
|
||||
#: hyphens and runs of spaces squashed to one space. A match marks the role as
|
||||
#: possibly remote (`remote_in_text`); it decides nothing by itself.
|
||||
REMOTE_TEXT: tuple[str, ...] = (
|
||||
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
|
||||
"remote working", "work remotely", "working remotely", "remote position", "remote role",
|
||||
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
|
||||
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
|
||||
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
|
||||
)
|
||||
REMOTE_TEXT_NOT: tuple[str, ...] = (
|
||||
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
|
||||
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
|
||||
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
|
||||
"remote control", "remote monitoring",
|
||||
)
|
||||
|
||||
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
|
||||
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
|
||||
_SQUASHED = re.compile(r"[\s-]+")
|
||||
|
||||
|
||||
def remote_in_text(title: str, text: str) -> bool:
|
||||
"""Whether the title or the text says the role is worked remotely: a
|
||||
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
|
||||
out. Cheap evidence, no model: a role marked so passes a location gate
|
||||
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
|
||||
false mark costs a cut slot at most, and a missed one nothing new."""
|
||||
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
|
||||
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
|
||||
|
||||
|
||||
def _line_word(text: str) -> str:
|
||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||
if found := remote_of(text):
|
||||
|
||||
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
-- 1 when the title or text says the role is worked remotely while no
|
||||
-- field does (prep, app.prep.words.remote_in_text): the location gate
|
||||
-- lets such a role through until extract has read it.
|
||||
remote_text INTEGER NOT NULL DEFAULT 0,
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
@@ -760,6 +764,8 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
# judge reads every role again (judge.FORMAT).
|
||||
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
||||
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
||||
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
|
||||
"location gate", lambda conn: _add_remote_text(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -1009,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||
|
||||
|
||||
def _add_remote_text(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
|
||||
which maps every open role again since its rules changed. Runs twice
|
||||
without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
if "remote_text" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
|
||||
|
||||
|
||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 27's step: `roles.remote_model` empty and
|
||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||
|
||||
+12
-4
@@ -355,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
return ("r.languages IN ('', '[]')",
|
||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||
if gate.field == "remote" and "remote" in values:
|
||||
# A role whose text says remote while no field does is possibly
|
||||
# remote (`remote_text`, prep): it meets a gate on remote until
|
||||
# extract has read it, and is not empty for it.
|
||||
return (f"({column} = '' AND r.remote_text = 0)",
|
||||
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
@@ -1625,8 +1631,9 @@ class Store:
|
||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country, remote_model, remote_where_model,
|
||||
deadline_host, deadline_model, deadline,
|
||||
employment, remote, remote_text, country, remote_model,
|
||||
remote_where_model, deadline_host, deadline_model, deadline,
|
||||
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
|
||||
json_extract(raw, '$.places') AS raw_places,
|
||||
(SELECT json_group_array(place_id) FROM (
|
||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||
@@ -1652,8 +1659,8 @@ class Store:
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
||||
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
|
||||
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||
the title's. `remote` is written as prep composed it (the model's,
|
||||
else the host's word, else the place line) and `remote_area`, place
|
||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||
@@ -1669,6 +1676,7 @@ class Store:
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = :remote,
|
||||
remote_text = :remote_text,
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
deadline = :deadline,
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
|
||||
Reference in new issue
Block a user