From d168d9eb6ce4ca83077db5d86d772fbdda62f28c Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 13:26:20 +0200 Subject: [PATCH] Add migration 26, roles.posting_url, kept across pulls and carried on the row and its copies Co-Authored-By: Claude Opus 5.5 --- app/models.py | 10 +++++++-- app/storage/schema.py | 14 ++++++++++++ app/storage/store.py | 10 +++++++-- tests/test_companies.py | 4 ++-- tests/test_dupes_storage.py | 2 +- tests/test_fetch_terms.py | 2 +- tests/test_overrides.py | 2 +- tests/test_prep.py | 2 +- tests/test_storage.py | 44 +++++++++++++++++++++++++++++++------ tests/test_web.py | 5 +++-- 10 files changed, 76 insertions(+), 19 deletions(-) diff --git a/app/models.py b/app/models.py index 5c6aa98..14a9e2e 100644 --- a/app/models.py +++ b/app/models.py @@ -809,6 +809,7 @@ class RoleCopy(Model): host: str = "" place: str = "" original_url: str = "" + posting_url: str = "" posted_at: str = "" @@ -865,6 +866,9 @@ class Role(Model): #: The short text the host's list carries, until `description` arrives. teaser: str = "" original_url: str = "" + #: Where a Jobindex ad's own page leads, found when the text is fetched + #: (`app.jobs.posting`); '' until then and for every other host. + posting_url: str = "" published_at: str #: The date this role counts as posted: see `posted_at`. Used for #: `recency` and the feed's newest order; `first_seen` stays what it @@ -974,8 +978,9 @@ class RoleRow(Model): there is one, else the description's opening, cut to `EXCERPT_LIMIT` characters. The pay fields are what the row's badge reads: the three normalised columns, the line the board stated, and `salary_state`, - derived as on `Role`. `original_url` is one short string, kept so the - `o` key opens a selected row's posting without opening the reader. + derived as on `Role`. `original_url` and `posting_url` are short strings, + kept so the `o` key opens a selected row's posting without opening the + reader. `seniority` is the title's level until extract reads the posting, '' when neither says. `copies` counts the open copies the feed hides behind this row.""" @@ -994,6 +999,7 @@ class RoleRow(Model): currency: str = "" salary_text: str = "" original_url: str = "" + posting_url: str = "" posted_at: str = "" score: float | None = None gated_reason: str = "" diff --git a/app/storage/schema.py b/app/storage/schema.py index 7fe1676..eb3fa72 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -69,6 +69,9 @@ CREATE TABLE IF NOT EXISTS roles ( description TEXT NOT NULL DEFAULT '', -- the posting's body text feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over original_url TEXT NOT NULL DEFAULT '', -- the posting page + -- Where a search host's own page leads: Jobindex's "Se jobbet" link, found + -- when the posting text is fetched (app.jobs.posting). Never written by an upsert. + posting_url TEXT NOT NULL DEFAULT '', published_at TEXT NOT NULL, -- what the source claims, UTC -- What storing postings buys. first_seen is when the app itself first -- stored the role; posted_at (app.models.posted_at) is the earlier of @@ -623,6 +626,9 @@ UPDATE roles SET seniority_model = '' UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''), NULLIF(seniority_stated, ''), seniority_title); """), + # A Python step: it adds the column only where it is missing. + (26, "roles.posting_url: where a Jobindex ad's own page leads, found when its text is " + "fetched", lambda conn: _add_posting_url(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -864,6 +870,14 @@ def _add_fetch_terms(conn: sqlite3.Connection) -> None: conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'") +def _add_posting_url(conn: sqlite3.Connection) -> None: + """Migration 26's step: `roles.posting_url`, empty for every role. Runs + twice without harm.""" + columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")} + if "posting_url" not in columns: + conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''") + + def _add_all_roles(conn: sqlite3.Connection) -> None: """Migration 21's step. The column; the tag for every company that is a favourite of any agent, since favourite no longer widens what a pull diff --git a/app/storage/store.py b/app/storage/store.py index 1e15076..9ecd17d 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -803,7 +803,7 @@ class Store: """The open copies of one kept role, oldest first; empty for a copy or a role without any. See `Role.copies`.""" return [RoleCopy.from_row(r) for r in self._rows( - f"""SELECT uid, {_HOST} AS host, place, original_url, posted_at FROM roles + f"""SELECT uid, {_HOST} AS host, place, original_url, posting_url, posted_at FROM roles WHERE dup_of = ? AND closed_at IS NULL AND {_NOT_A_PLACE.format(kept="?")} ORDER BY posted_at, uid""", (uid, uid))] @@ -1060,7 +1060,7 @@ class Store: "SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, " "roles.country, roles.remote, roles.function, roles.seniority, roles.salary_min, " "roles.salary_max, roles.currency, roles.salary_text, roles.original_url, " - "roles.posted_at, roles.read_at, " + "roles.posting_url, roles.posted_at, roles.read_at, " "v.vote, v.voted_at, " f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt") order = "roles.posted_at DESC, roles.uid" @@ -1629,6 +1629,12 @@ class Store: return None return self.role(uid) + def set_posting_url(self, uid: str, url: str) -> bool: + """Keep where a search host's own page leads. False for an unknown + uid. Only this writes it: `upsert_role` leaves it alone, so a pull + keeps it.""" + return bool(self._write("UPDATE roles SET posting_url = ? WHERE uid = ?", (url, uid))) + # --------------------------------------------------------------- enrich def enrich_waiting(self, top: int, floor: float, diff --git a/tests/test_companies.py b/tests/test_companies.py index 63b04fc..979fb94 100644 --- a/tests/test_companies.py +++ b/tests/test_companies.py @@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) -> "VALUES (?, ?, '', '')", (key, name)) conn.execute("PRAGMA user_version = 19") reopened = Store(path) - assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25] + assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26] assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])] assert len(reopened.companies()) == 3 @@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp conn.execute("ALTER TABLE companies DROP COLUMN all_countries") conn.execute("PRAGMA user_version = 20") reopened = Store(path) - assert reopened.migrations_applied == [21, 22, 23, 24, 25] + assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26] # The tag of 21 is All countries and All roles since 24. tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0} assert {c.key: c.all_roles for c in reopened.companies()} == tagged diff --git a/tests/test_dupes_storage.py b/tests/test_dupes_storage.py index b3bad32..c462cf6 100644 --- a/tests/test_dupes_storage.py +++ b/tests/test_dupes_storage.py @@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26] assert dup_of(migrated) == {first.uid: None, second.uid: first.uid} with sqlite3.connect(path) as conn: assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'" diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index 100c533..8efdc97 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -437,7 +437,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None: conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms") conn.execute("PRAGMA user_version = 22") migrated = Store(path) - assert migrated.migrations_applied == [23, 24, 25] + assert migrated.migrations_applied == [23, 24, 25, 26] assert migrated.agents()[0].fetch_terms == [] diff --git a/tests/test_overrides.py b/tests/test_overrides.py index 91654da..e2b75e9 100644 --- a/tests/test_overrides.py +++ b/tests/test_overrides.py @@ -120,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'") conn.execute("PRAGMA user_version = 23") migrated = Store(path) - assert migrated.migrations_applied == [24, 25] + assert migrated.migrations_applied == [24, 25, 26] flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()} assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)} params = {s.target: s.params for s in migrated.subscriptions(source="ats")} diff --git a/tests/test_prep.py b/tests/test_prep.py index 462fc4e..452c627 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -314,7 +314,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None: migrated = Store(path) - assert migrated.migrations_applied == [22, 23, 24, 25] + assert migrated.migrations_applied == [22, 23, 24, 25, 26] one = row(migrated, "1") assert (one["seniority_host"], one["employment_host"], one["remote_host"], one["country_host"]) == ("senior", "full_time", "true", "DK") diff --git a/tests/test_storage.py b/tests/test_storage.py index 1371802..a3b3292 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 25 + assert LATEST == 26 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated( migrated = Store(path) - assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] acme = migrated.company("acme") assert acme is not None assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example") @@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P migrated = Store(path) - assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr] Prompt(name="Backend", text="python backend")] assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr] @@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None migrated = Store(path) - assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] acme = migrated.company("acme") assert acme is not None assert (acme.careers_url, acme.notes, acme.hand_reason) == ( @@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read( migrated = Store(path) - assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26] with sqlite3.connect(path) as conn: rows = {uid: (fold, level) for uid, fold, level in conn.execute( "SELECT uid, place_fold, seniority FROM roles")} @@ -688,7 +688,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) -> migrated = Store(path) - assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25] + assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26] assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"] assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"] assert migrated.role_places(stated.uid) == [] @@ -730,7 +730,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level( migrated = Store(path) - assert migrated.migrations_applied == [25] + assert migrated.migrations_applied == [25, 26] with sqlite3.connect(path) as conn: found = {sid: rest for sid, *rest in conn.execute( "SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority " @@ -738,6 +738,36 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level( assert found == {sid: list(after) for sid, (_, after) in cases.items()} +def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path) -> None: + path = tmp_path / "v25.db" + store = Store(path) + upsert(store, posting("a")) + with sqlite3.connect(path) as conn: + conn.execute("ALTER TABLE roles DROP COLUMN posting_url") + conn.execute("PRAGMA user_version = 25") + + migrated = Store(path) + + assert migrated.migrations_applied == [26] + role = migrated.role(posting("a").uid) + assert role is not None and role.posting_url == "" + with sqlite3.connect(path) as conn: + conn.execute("PRAGMA user_version = 25") # the column is there already + assert Store(path).migrations_applied == [26] + + +def test_a_pull_keeps_the_posting_url(store: Store) -> None: + upsert(store, posting("a")) + uid = posting("a").uid + assert store.set_posting_url(uid, "https://careers.example/jobs/1") + assert not store.set_posting_url("nope", "https://careers.example/jobs/1") + upsert(store, posting("a")) # the next pull reads the same role again + role = store.role(uid) + assert role is not None and role.posting_url == "https://careers.example/jobs/1" + row = next(r for r in store.roles_page() if r.uid == uid) + assert row.posting_url == "https://careers.example/jobs/1" + + def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: upsert(store, posting(place="Berlin", remote=True, salary_min="80000", salary_max=95000, currency="EUR", department="Eng")) diff --git a/tests/test_web.py b/tests/test_web.py index 0ba1b6e..6e28d3e 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1224,7 +1224,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns( assert set(row) == { "uid", "company_key", "company", "title", "place", "country", "remote", "function", "seniority", "salary_min", "salary_max", "currency", "salary_text", "salary_state", - "original_url", "posted_at", "score", "gated_reason", "prompt", "vote", + "original_url", "posting_url", "posted_at", "score", "gated_reason", "prompt", "vote", "voted_at", "read_at", "tags", "excerpt", "copies", } assert row["uid"] == uid @@ -1734,7 +1734,8 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role( assert kept["dup_of"] is None assert kept["copies"] == [{ "uid": search.uid, "host": "jobindex", "place": "Ballerup, Denmark", - "original_url": search.location, "posted_at": kept["copies"][0]["posted_at"]}] + "original_url": search.location, "posting_url": "", + "posted_at": kept["copies"][0]["posted_at"]}] copy = client.get(f"/api/roles/{search.uid}").json() assert copy["dup_of"] == board.uid and copy["copies"] == []