Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
287 lines
13 KiB
Python
287 lines
13 KiB
Python
"""`roles.dup_of`: migration 18, `Store.rebuild_dupes`, and what a copy is left out of."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from desk.models import Item
|
|
|
|
from app.storage import Store
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def role(source_id: str, *, company: str = "SAP", title: str = "Forward Deployed Engineer",
|
|
place: str = "Garching", body: str = "The posting.", day: int = 20,
|
|
**raw: object) -> Item:
|
|
item = Item(source_id=source_id, location=f"https://jobs.example/{source_id}",
|
|
title=title, body=body, published_at=datetime(2026, 9, day, tzinfo=UTC),
|
|
raw={"company": company, "place": place, "board": "successfactors", **raw})
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
def dup_of(store: Store) -> dict[str, str | None]:
|
|
with sqlite3.connect(store.path) as conn:
|
|
return dict(conn.execute("SELECT uid, dup_of FROM roles"))
|
|
|
|
|
|
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
|
|
store: Store) -> None:
|
|
"""SAP's repeats: three Garching rows are one kept role (the one with
|
|
the description) and two copies; New York, listed by another host, is its own role."""
|
|
garching = [role("1", body=""), role("2"), role("3", body="")]
|
|
york = role("4", place="New York, NY, US", board="greenhouse")
|
|
for item in (*garching, york):
|
|
store.upsert_role(item)
|
|
|
|
assert store.rebuild_dupes() == (1, 2, 2)
|
|
kept = garching[1].uid
|
|
assert dup_of(store) == {garching[0].uid: kept, kept: None,
|
|
garching[2].uid: kept, york.uid: None}
|
|
assert store.rebuild_dupes().changed == 0 # stable between runs
|
|
|
|
|
|
def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) -> None:
|
|
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
|
|
board="teamtailor", body="")
|
|
search = role("search", company="Trackunit", title="Backend Engineer",
|
|
place="Aalborg, Denmark", board=None, provider="jobindex")
|
|
for item in (search, board):
|
|
store.upsert_role(item)
|
|
store.rebuild_dupes()
|
|
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
|
|
|
|
|
|
def test_a_freehire_copy_of_a_greenhouse_role_is_hidden_behind_the_board(
|
|
store: Store, search_stubs: dict[str, object]) -> None:
|
|
"""freehire aggregates other hosts' postings; the same role it lists from
|
|
a board roledesk reads is one role, and the board's copy is the one kept
|
|
even though freehire's is the one with a description."""
|
|
board = role("board", company="Lunar", title="Head of Data", place="Copenhagen",
|
|
board="greenhouse", body="", day=22)
|
|
copy = role("copy", company="Lunar", title="Head of Data", place="Copenhagen, Denmark",
|
|
board=None, provider="freehire", source="greenhouse", day=20)
|
|
for item in (copy, board):
|
|
store.upsert_role(item)
|
|
store.rebuild_dupes()
|
|
assert dup_of(store) == {board.uid: None, copy.uid: board.uid}
|
|
|
|
|
|
def kept_of(store: Store, *items: Item) -> str:
|
|
"""The uid of the one of these roles that is not a copy."""
|
|
found = dup_of(store)
|
|
(kept,) = [i.uid for i in items if found[i.uid] is None]
|
|
return kept
|
|
|
|
|
|
def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
|
|
store: Store) -> None:
|
|
"""One posting per place on one host is one role that lists both places,
|
|
each with its own link; the count of copies does not show it as `x2`."""
|
|
berlin, munich = role("b", place="Berlin"), role("m", place="Munich")
|
|
for item in (berlin, munich):
|
|
store.upsert_role(item)
|
|
assert store.rebuild_dupes() == (1, 1, 1)
|
|
kept = kept_of(store, berlin, munich)
|
|
|
|
got = store.role(kept)
|
|
assert got is not None
|
|
assert sorted(p.text for p in got.places) == ["Berlin", "Munich"]
|
|
assert sorted(p.url for p in got.places if p.url) in (
|
|
["https://jobs.example/b"], ["https://jobs.example/m"])
|
|
assert got.copies == []
|
|
(row,) = store.roles_page()
|
|
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
|
|
|
|
|
|
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
|
|
a = role("a", place="Berlin", body="Build payment services in Kotlin for our bank " * 5)
|
|
b = role("b", place="Munich", body="Sell cloud contracts to enterprise buyers " * 5)
|
|
c = role("c", place="Hamburg", board="greenhouse", body=a.body)
|
|
for item in (a, b, c):
|
|
store.upsert_role(item)
|
|
assert store.rebuild_dupes() == (0, 0, 0)
|
|
|
|
|
|
def own_and_copy_places(store: Store, uid: str) -> list[tuple[str, bool]]:
|
|
got = store.role(uid)
|
|
assert got is not None
|
|
return sorted((p.text, bool(p.url)) for p in got.places)
|
|
|
|
|
|
def test_a_place_copy_closing_or_its_board_off_takes_its_place_away(store: Store) -> None:
|
|
from app.jobs.sources import set_fetch
|
|
sub = store.add_subscription("ats", "successfactors:sap", label="SAP")
|
|
items = [role("b", place="Berlin"), role("m", place="Munich"), role("k", place="Kiel")]
|
|
for item in items:
|
|
store.upsert_role(item, sub)
|
|
store.rebuild_dupes()
|
|
kept = kept_of(store, *items)
|
|
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Kiel", "Munich"]
|
|
|
|
closing = next(i for i in items if i.uid != kept)
|
|
with sqlite3.connect(store.path) as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (closing.uid,))
|
|
store.rebuild_dupes()
|
|
assert len(own_and_copy_places(store, kept)) == 2
|
|
assert store.rebuild_dupes().changed == 0
|
|
|
|
set_fetch(store, sub.id, False)
|
|
assert set(dup_of(store).values()) == {None}
|
|
assert not any(url for _, url in own_and_copy_places(store, kept))
|
|
|
|
|
|
def test_a_repull_keeps_the_copies_places(store: Store) -> None:
|
|
items = [role("b", place="Berlin"), role("m", place="Munich")]
|
|
for item in items:
|
|
store.upsert_role(item)
|
|
store.rebuild_dupes()
|
|
kept = kept_of(store, *items)
|
|
store.upsert_role(next(i for i in items if i.uid == kept))
|
|
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Munich"]
|
|
|
|
|
|
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
|
|
store: Store) -> None:
|
|
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
|
|
for item in (first, second, third):
|
|
store.upsert_role(item)
|
|
store.rebuild_dupes()
|
|
assert dup_of(store)[second.uid] == first.uid
|
|
|
|
with sqlite3.connect(store.path) as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
|
|
store.rebuild_dupes()
|
|
assert dup_of(store) == {first.uid: None, second.uid: None, third.uid: second.uid}
|
|
|
|
with sqlite3.connect(store.path) as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-26' WHERE uid = ?", (third.uid,))
|
|
store.rebuild_dupes()
|
|
assert set(dup_of(store).values()) == {None}
|
|
|
|
|
|
def test_read_and_votes_move_to_the_kept_role(store: Store) -> None:
|
|
"""A copy read and liked before it was known as one: the kept role
|
|
becomes read and liked. An agent that voted on the kept role keeps its
|
|
own vote, and a later unread on the kept role is not undone."""
|
|
one, two = store.save_agent("One"), store.save_agent("Two")
|
|
kept, copy = role("kept", day=18), role("copy", day=19)
|
|
store.upsert_role(copy)
|
|
store.set_vote(one.id, copy.uid, 1)
|
|
store.set_vote(two.id, copy.uid, -1)
|
|
store.upsert_role(kept)
|
|
store.set_vote(two.id, kept.uid, 0)
|
|
|
|
store.rebuild_dupes()
|
|
|
|
read = store.role(copy.uid).read_at # type: ignore[union-attr]
|
|
assert store.role(kept.uid).read_at == read # type: ignore[union-attr]
|
|
assert store.votes(one.id) == {kept.uid: 1}
|
|
assert store.votes(two.id) == {kept.uid: 0, copy.uid: -1}
|
|
|
|
store.set_read(kept.uid, False)
|
|
store.rebuild_dupes()
|
|
assert store.role(kept.uid).read_at is None # type: ignore[union-attr]
|
|
|
|
|
|
def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> None:
|
|
agent = store.save_agent("One")
|
|
first, second = role("1", day=18), role("2", day=19)
|
|
for item in (first, second):
|
|
store.upsert_role(item)
|
|
store.rebuild_dupes()
|
|
store.set_vote(agent.id, first.uid, 1)
|
|
with sqlite3.connect(store.path) as conn:
|
|
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
|
|
# A third copy keeps the group alive once the first has closed.
|
|
store.upsert_role(role("3", day=20))
|
|
|
|
store.rebuild_dupes()
|
|
|
|
assert store.votes(agent.id) == {second.uid: 1}
|
|
assert store.role(second.uid).read_at is not None # type: ignore[union-attr]
|
|
|
|
|
|
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
|
|
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin", board="greenhouse")
|
|
for item in (kept, copy, other):
|
|
store.upsert_role(item)
|
|
store.set_tags(item.uid, ["python"])
|
|
store.rebuild_dupes()
|
|
agent = store.save_agent("A", intent="engineering")
|
|
|
|
assert {r.uid for r in store.roles()} == {kept.uid, other.uid}
|
|
rows = {r.uid: r.copies for r in store.roles_page()}
|
|
assert rows == {kept.uid: 1, other.uid: 0}
|
|
assert store.count_roles() == 2
|
|
assert sum(n for _, n in store.role_facets()["seniority"]) == 2
|
|
assert store.tag_counts() == [("python", 2, 2)]
|
|
assert {c.uid for c in store.score_candidates(agent.id, [])} == {kept.uid, other.uid}
|
|
assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid}
|
|
assert store.open_role_counts() == {"sap": 2}
|
|
assert [count for _, _, count, _ in store.open_companies()] == [2]
|
|
store.replace_scores(agent.id, [(u, 0.9, "") for u in (kept.uid, copy.uid, other.uid)])
|
|
assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid}
|
|
assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid}
|
|
|
|
# The copy still opens by uid, and the kept role lists it.
|
|
assert store.role(copy.uid).dup_of == kept.uid # type: ignore[union-attr]
|
|
assert [c.uid for c in store.role(kept.uid).copies] == [copy.uid] # type: ignore[union-attr]
|
|
assert store.role(other.uid).copies == [] # type: ignore[union-attr]
|
|
|
|
|
|
def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
|
path = tmp_path / "v17.db"
|
|
store = Store(path)
|
|
first, second = role("1", day=18), role("2", day=19)
|
|
for item in (first, second):
|
|
store.upsert_role(item)
|
|
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
|
|
with sqlite3.connect(path) as conn:
|
|
for table in ("role_places", "place_includes", "place_aliases", "places"):
|
|
conn.execute(f"DROP TABLE {table}")
|
|
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
|
|
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
|
conn.execute("DROP TABLE company_aliases")
|
|
for column in ("sector", "stage", "size_band", "founded", "last_funding",
|
|
"last_funding_currency", "facts_source", "facts_at"):
|
|
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
|
conn.execute("DROP INDEX idx_roles_dup_of")
|
|
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
|
|
conn.execute("PRAGMA user_version = 17")
|
|
|
|
migrated = Store(path)
|
|
|
|
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
|
|
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
|
with sqlite3.connect(path) as conn:
|
|
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
|
).fetchone() is not None
|
|
|
|
|
|
def test_switching_a_board_off_shows_its_roles_copies_at_once(store: Store) -> None:
|
|
"""A close outside a pull regroups too: the search's copy of a role whose
|
|
board is switched off is listed again without waiting for a pull."""
|
|
from app.jobs.sources import set_fetch
|
|
sub = store.add_subscription("ats", "teamtailor:trackunit", label="Trackunit")
|
|
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
|
|
board="teamtailor")
|
|
search = role("search", company="Trackunit", title="Backend Engineer",
|
|
place="Aalborg, Denmark", board=None, provider="jobindex")
|
|
store.upsert_role(board, sub)
|
|
store.upsert_role(search)
|
|
store.rebuild_dupes()
|
|
assert dup_of(store)[search.uid] == board.uid
|
|
|
|
set_fetch(store, sub.id, False)
|
|
assert dup_of(store) == {board.uid: None, search.uid: None}
|