Mark a role whose title or text says remote while no field does, in prep with no model, and let it meet a gate on remote until extract has read it (migration 35, roles.remote_text)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
f141cffc3c
commit
49929372c7
14 files changed
+144
-36
No files matched your search
+6
-2
@@ -37,8 +37,9 @@ from app.storage import Store
|
|||||||
#: Raised when the rules in this package change the way a stored word maps,
|
#: Raised when the rules in this package change the way a stored word maps,
|
||||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||||
#: remote area, and remote as the model's before the host's word. 4: the
|
#: remote area, and remote as the model's before the host's word. 4: the
|
||||||
#: deadline, the host's before the model's.
|
#: deadline, the host's before the model's. 5: `remote_text`, the title or
|
||||||
VERSION = 4
|
#: text saying remote while no field does.
|
||||||
|
VERSION = 5
|
||||||
|
|
||||||
#: The state key the last run's `fingerprint` is kept under.
|
#: The state key the last run's `fingerprint` is kept under.
|
||||||
STATE = "prep:tables"
|
STATE = "prep:tables"
|
||||||
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
|
|||||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||||
code describes and `VERSION`."""
|
code describes and `VERSION`."""
|
||||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||||
|
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
|
||||||
"places": repr(places.seed())}, sort_keys=True)
|
"places": repr(places.seed())}, sort_keys=True)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()
|
return hashlib.sha1(blob.encode()).hexdigest()
|
||||||
|
|
||||||
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
|||||||
"seniority_title": titled,
|
"seniority_title": titled,
|
||||||
"employment": words.employment(host, row["employment_host"], tables),
|
"employment": words.employment(host, row["employment_host"], tables),
|
||||||
"remote": remote,
|
"remote": remote,
|
||||||
|
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
|
||||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||||
if remote == "remote" else ()),
|
if remote == "remote" else ()),
|
||||||
"country": country(gaz, place_ids, row["country_host"]),
|
"country": country(gaz, place_ids, row["country_host"]),
|
||||||
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
|||||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||||
or found["remote"] != row["remote"] \
|
or found["remote"] != row["remote"] \
|
||||||
|
or found["remote_text"] != row["remote_text"] \
|
||||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||||
or (found["country"] and found["country"] != row["country"]) \
|
or (found["country"] and found["country"] != row["country"]) \
|
||||||
or found["deadline"] != row["deadline"] \
|
or found["deadline"] != row["deadline"] \
|
||||||
|
|||||||
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
|
|||||||
re.IGNORECASE)
|
re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
#: Phrases in a title or text that say the role is worked remotely, and the
|
||||||
|
#: phrases that take one back. Both are read on the lowercased text with its
|
||||||
|
#: hyphens and runs of spaces squashed to one space. A match marks the role as
|
||||||
|
#: possibly remote (`remote_in_text`); it decides nothing by itself.
|
||||||
|
REMOTE_TEXT: tuple[str, ...] = (
|
||||||
|
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
|
||||||
|
"remote working", "work remotely", "working remotely", "remote position", "remote role",
|
||||||
|
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
|
||||||
|
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
|
||||||
|
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
|
||||||
|
)
|
||||||
|
REMOTE_TEXT_NOT: tuple[str, ...] = (
|
||||||
|
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
|
||||||
|
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
|
||||||
|
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
|
||||||
|
"remote control", "remote monitoring",
|
||||||
|
)
|
||||||
|
|
||||||
|
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
|
||||||
|
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
|
||||||
|
_SQUASHED = re.compile(r"[\s-]+")
|
||||||
|
|
||||||
|
|
||||||
|
def remote_in_text(title: str, text: str) -> bool:
|
||||||
|
"""Whether the title or the text says the role is worked remotely: a
|
||||||
|
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
|
||||||
|
out. Cheap evidence, no model: a role marked so passes a location gate
|
||||||
|
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
|
||||||
|
false mark costs a cut slot at most, and a missed one nothing new."""
|
||||||
|
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
|
||||||
|
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
|
||||||
|
|
||||||
|
|
||||||
def _line_word(text: str) -> str:
|
def _line_word(text: str) -> str:
|
||||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||||
if found := remote_of(text):
|
if found := remote_of(text):
|
||||||
|
|||||||
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
|
|||||||
country TEXT NOT NULL DEFAULT '',
|
country TEXT NOT NULL DEFAULT '',
|
||||||
remote TEXT NOT NULL DEFAULT ''
|
remote TEXT NOT NULL DEFAULT ''
|
||||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||||
|
-- 1 when the title or text says the role is worked remotely while no
|
||||||
|
-- field does (prep, app.prep.words.remote_in_text): the location gate
|
||||||
|
-- lets such a role through until extract has read it.
|
||||||
|
remote_text INTEGER NOT NULL DEFAULT 0,
|
||||||
salary_min INTEGER,
|
salary_min INTEGER,
|
||||||
salary_max INTEGER,
|
salary_max INTEGER,
|
||||||
currency TEXT NOT NULL DEFAULT '',
|
currency TEXT NOT NULL DEFAULT '',
|
||||||
@@ -760,6 +764,8 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
|||||||
# judge reads every role again (judge.FORMAT).
|
# judge reads every role again (judge.FORMAT).
|
||||||
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
||||||
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
||||||
|
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
|
||||||
|
"location gate", lambda conn: _add_remote_text(conn)),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: What a file created from `SCHEMA` today is already at.
|
#: What a file created from `SCHEMA` today is already at.
|
||||||
@@ -1009,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
|||||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||||
|
|
||||||
|
|
||||||
|
def _add_remote_text(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
|
||||||
|
which maps every open role again since its rules changed. Runs twice
|
||||||
|
without harm."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||||
|
if "remote_text" not in columns:
|
||||||
|
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
|
||||||
|
|
||||||
|
|
||||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||||
"""Migration 27's step: `roles.remote_model` empty and
|
"""Migration 27's step: `roles.remote_model` empty and
|
||||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||||
|
|||||||
+12
-4
@@ -355,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
return ("r.languages IN ('', '[]')",
|
return ("r.languages IN ('', '[]')",
|
||||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||||
|
if gate.field == "remote" and "remote" in values:
|
||||||
|
# A role whose text says remote while no field does is possibly
|
||||||
|
# remote (`remote_text`, prep): it meets a gate on remote until
|
||||||
|
# extract has read it, and is not empty for it.
|
||||||
|
return (f"({column} = '' AND r.remote_text = 0)",
|
||||||
|
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
|
||||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||||
|
|
||||||
|
|
||||||
@@ -1625,8 +1631,9 @@ class Store:
|
|||||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||||
seniority_host, employment_host, remote_host, country_host,
|
seniority_host, employment_host, remote_host, country_host,
|
||||||
seniority_model, seniority_stated, seniority_title, seniority,
|
seniority_model, seniority_stated, seniority_title, seniority,
|
||||||
employment, remote, country, remote_model, remote_where_model,
|
employment, remote, remote_text, country, remote_model,
|
||||||
deadline_host, deadline_model, deadline,
|
remote_where_model, deadline_host, deadline_model, deadline,
|
||||||
|
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
|
||||||
json_extract(raw, '$.places') AS raw_places,
|
json_extract(raw, '$.places') AS raw_places,
|
||||||
(SELECT json_group_array(place_id) FROM (
|
(SELECT json_group_array(place_id) FROM (
|
||||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||||
@@ -1652,8 +1659,8 @@ class Store:
|
|||||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||||
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
|
||||||
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||||
the title's. `remote` is written as prep composed it (the model's,
|
the title's. `remote` is written as prep composed it (the model's,
|
||||||
else the host's word, else the place line) and `remote_area`, place
|
else the host's word, else the place line) and `remote_area`, place
|
||||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||||
@@ -1669,6 +1676,7 @@ class Store:
|
|||||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||||
employment = :employment,
|
employment = :employment,
|
||||||
remote = :remote,
|
remote = :remote,
|
||||||
|
remote_text = :remote_text,
|
||||||
country = COALESCE(NULLIF(:country, ''), country),
|
country = COALESCE(NULLIF(:country, ''), country),
|
||||||
deadline = :deadline,
|
deadline = :deadline,
|
||||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
|||||||
"VALUES (?, ?, '', '')", (key, name))
|
"VALUES (?, ?, '', '')", (key, name))
|
||||||
conn.execute("PRAGMA user_version = 19")
|
conn.execute("PRAGMA user_version = 19")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||||
assert len(reopened.companies()) == 3
|
assert len(reopened.companies()) == 3
|
||||||
|
|
||||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
|||||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||||
conn.execute("PRAGMA user_version = 20")
|
conn.execute("PRAGMA user_version = 20")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
# The tag of 21 is All countries and All roles since 24.
|
# The tag of 21 is All countries and All roles since 24.
|
||||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
rows = {r["source_id"]: r for r in migrated._rows(
|
rows = {r["source_id"]: r for r in migrated._rows(
|
||||||
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
||||||
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
||||||
@@ -106,7 +106,7 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
|||||||
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 27")
|
conn.execute("PRAGMA user_version = 27")
|
||||||
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33, 34]
|
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- deadline
|
# ---------------------------------------------------------------- deadline
|
||||||
@@ -260,7 +260,7 @@ def test_v34_drops_the_must_haves_and_runs_twice(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
for _ in range(2): # runs twice without harm
|
for _ in range(2): # runs twice without harm
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [34]
|
assert migrated.migrations_applied == [34, 35]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert "must_text" not in {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
assert "must_text" not in {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
||||||
assert not {"musts", "broken"} & {
|
assert not {"musts", "broken"} & {
|
||||||
@@ -280,12 +280,12 @@ def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||||
[data] = migrated.agents()
|
[data] = migrated.agents()
|
||||||
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 29")
|
conn.execute("PRAGMA user_version = 29")
|
||||||
assert Store(path).migrations_applied == [30, 31, 32, 33, 34]
|
assert Store(path).migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
||||||
@@ -313,7 +313,7 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
|||||||
|
|
||||||
for _ in range(2): # runs twice without harm
|
for _ in range(2): # runs twice without harm
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [33, 34]
|
assert migrated.migrations_applied == [33, 34, 35]
|
||||||
assert migrated.judgement(old.id, item.uid) is None
|
assert migrated.judgement(old.id, item.uid) is None
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
|
|||||||
@@ -282,7 +282,7 @@ def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Pat
|
|||||||
conn.execute("DROP TABLE agent_cv_vectors")
|
conn.execute("DROP TABLE agent_cv_vectors")
|
||||||
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
||||||
conn.execute("PRAGMA user_version = 30")
|
conn.execute("PRAGMA user_version = 30")
|
||||||
assert Store(path).migrations_applied == [31, 32, 33, 34]
|
assert Store(path).migrations_applied == [31, 32, 33, 34, 35]
|
||||||
step = {v: s for v, _, s in MIGRATIONS}[31]
|
step = {v: s for v, _, s in MIGRATIONS}[31]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert callable(step)
|
assert callable(step)
|
||||||
|
|||||||
@@ -281,7 +281,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||||
|
|||||||
@@ -520,7 +520,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
|||||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||||
conn.execute("PRAGMA user_version = 22")
|
conn.execute("PRAGMA user_version = 22")
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.agents()[0].fetch_terms == []
|
assert migrated.agents()[0].fetch_terms == []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("countries, roles, line", [
|
@pytest.mark.parametrize("countries, roles, line", [
|
||||||
(False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
(False, False, "follows agents' world (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||||
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
|
(False, True, "follows agents' world (Europe) · remote not asked on Phenom · all roles"),
|
||||||
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||||
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
|
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
|
||||||
], ids=["neither", "all roles alone", "all locations alone", "both"])
|
], ids=["neither", "all roles alone", "all locations alone", "both"])
|
||||||
@@ -122,7 +122,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
|
|||||||
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
||||||
conn.execute("PRAGMA user_version = 23")
|
conn.execute("PRAGMA user_version = 23")
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
||||||
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
||||||
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
||||||
|
|||||||
+26
-2
@@ -224,7 +224,31 @@ def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
|||||||
("senior", "lead", "onsite")
|
("senior", "lead", "onsite")
|
||||||
|
|
||||||
|
|
||||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
def test_remote_in_the_text_is_marked_and_a_denial_is_not() -> None:
|
||||||
|
yes = ("Fully-remote within the EU, we meet twice a year.",
|
||||||
|
"Work from home or from our Berlin office, as you like.",
|
||||||
|
"Dette er en stilling med mulighed for fjernarbejde.")
|
||||||
|
no = ("No remote work: this role is on site in Munich.",
|
||||||
|
"Remote sensing analyst for our satellite team.",
|
||||||
|
"A great office in Copenhagen with free lunch.")
|
||||||
|
assert all(words.remote_in_text("Engineer", text) for text in yes)
|
||||||
|
assert not any(words.remote_in_text("Engineer", text) for text in no)
|
||||||
|
assert words.remote_in_text("Backend Developer, remote-first team", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_prep_marks_a_role_whose_text_says_remote(store: Store) -> None:
|
||||||
|
said = item("1", title="Engineer", place="Berlin")
|
||||||
|
said.body = "Fully remote within Germany, with a Berlin office."
|
||||||
|
silent = item("2", title="Engineer", place="Berlin")
|
||||||
|
silent.body = "On site in Berlin."
|
||||||
|
store.upsert_role(said)
|
||||||
|
store.upsert_role(silent)
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
assert (row(store, "1")["remote"], row(store, "1")["remote_text"]) == ("", 1)
|
||||||
|
assert (row(store, "2")["remote"], row(store, "2")["remote_text"]) == ("", 0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.upsert_role(item("1", company="ORSTED A/S"))
|
store.upsert_role(item("1", company="ORSTED A/S"))
|
||||||
assert row(store, "1")["company_key"] != "ørsted"
|
assert row(store, "1")["company_key"] != "ørsted"
|
||||||
@@ -316,7 +340,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
one = row(migrated, "1")
|
one = row(migrated, "1")
|
||||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||||
|
|||||||
@@ -534,7 +534,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.gazetteer().anywhere is not None
|
assert migrated.gazetteer().anywhere is not None
|
||||||
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
||||||
"WHERE uid = ?", (board,))
|
"WHERE uid = ?", (board,))
|
||||||
@@ -544,7 +544,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
|||||||
assert area(migrated, search.uid) == ("", "")
|
assert area(migrated, search.uid) == ("", "")
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 26")
|
conn.execute("PRAGMA user_version = 26")
|
||||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34]
|
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
||||||
|
|||||||
@@ -417,6 +417,30 @@ def test_a_region_that_holds_the_agents_places_is_no_evidence_against(
|
|||||||
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
|
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_whose_text_says_remote_passes_a_gate_on_remote_until_extract_reads_it(
|
||||||
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
|
"""Berlin is evidence against Denmark, but a text that says remote while
|
||||||
|
no field does makes the role possibly remote (`remote_text`, prep): it
|
||||||
|
meets the remote member. Once extract says onsite, it fails again."""
|
||||||
|
said = role(store, "1", place="Berlin, Germany", remote="")
|
||||||
|
silent = role(store, "2", place="Berlin, Germany", remote="")
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET remote_text = 1 WHERE uid = ?", (said,))
|
||||||
|
agent = store.save_agent("DK or remote", "", [
|
||||||
|
Gate(field="country", values=["DK"], mode="any"),
|
||||||
|
Gate(field="remote", values=["remote"], mode="any")])
|
||||||
|
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
scores = store.scores(agent.id)
|
||||||
|
assert scores[said][1] == ""
|
||||||
|
assert scores[silent] == (0.0, "none of country DK, remote remote")
|
||||||
|
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET remote = 'onsite' WHERE uid = ?", (said,))
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
assert store.scores(agent.id)[said] == (0.0, "none of country DK, remote remote")
|
||||||
|
|
||||||
|
|
||||||
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
|
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
|
||||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
"""Europe alone is no evidence against Denmark; `London; Europe` is in
|
"""Europe alone is no evidence against Denmark; `London; Europe` is in
|
||||||
|
|||||||
+12
-12
@@ -71,7 +71,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
|||||||
"application_events", "agents", "scores", "state"} <= tables
|
"application_events", "agents", "scores", "state"} <= tables
|
||||||
assert "idx_roles_cluster" in indexes
|
assert "idx_roles_cluster" in indexes
|
||||||
assert store.migrations_applied == []
|
assert store.migrations_applied == []
|
||||||
assert LATEST == 34
|
assert LATEST == 35
|
||||||
# A second open finds it current and applies nothing.
|
# A second open finds it current and applies nothing.
|
||||||
assert Store(path).migrations_applied == []
|
assert Store(path).migrations_applied == []
|
||||||
|
|
||||||
@@ -191,7 +191,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||||
@@ -396,7 +396,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||||
Prompt(name="Backend", text="python backend")]
|
Prompt(name="Backend", text="python backend")]
|
||||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||||
@@ -420,7 +420,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||||
@@ -449,7 +449,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||||
"SELECT uid, place_fold, seniority FROM roles")}
|
"SELECT uid, place_fold, seniority FROM roles")}
|
||||||
@@ -691,7 +691,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||||
assert migrated.role_places(stated.uid) == []
|
assert migrated.role_places(stated.uid) == []
|
||||||
@@ -733,7 +733,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
found = {sid: rest for sid, *rest in conn.execute(
|
found = {sid: rest for sid, *rest in conn.execute(
|
||||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||||
@@ -751,12 +751,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
role = migrated.role(posting("a").uid)
|
role = migrated.role(posting("a").uid)
|
||||||
assert role is not None and role.posting_url == ""
|
assert role is not None and role.posting_url == ""
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34]
|
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||||
@@ -1855,7 +1855,7 @@ def test_v32_resets_the_limits_drops_prefer_gates_and_renames_enrich(tmp_path: P
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [32, 33, 34]
|
assert migrated.migrations_applied == [32, 33, 34, 35]
|
||||||
by_name = {a.name: a for a in migrated.agents()}
|
by_name = {a.name: a for a in migrated.agents()}
|
||||||
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
||||||
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
||||||
@@ -1880,14 +1880,14 @@ def test_v32_runs_twice_and_keeps_limits_set_since(tmp_path: Path) -> None:
|
|||||||
store = Store(path)
|
store = Store(path)
|
||||||
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
||||||
_v31(path)
|
_v31(path)
|
||||||
assert Store(path).migrations_applied == [32, 33, 34]
|
assert Store(path).migrations_applied == [32, 33, 34, 35]
|
||||||
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
||||||
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
||||||
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
||||||
again = Store(path)
|
again = Store(path)
|
||||||
assert again.migrations_applied == [32, 33, 34]
|
assert again.migrations_applied == [32, 33, 34, 35]
|
||||||
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user