Files
jobboard/tests/test_dupes_storage.py
T

341 lines
15 KiB
Python

"""`roles.dup_of`: migration 18, `Store.rebuild_dupes`, and what a copy is left out of."""
from __future__ import annotations
import sqlite3
from datetime import UTC, datetime
from pathlib import Path
import numpy as np
import pytest
from desk.models import Item
from app.storage import Store
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
def role(source_id: str, *, company: str = "SAP", title: str = "Forward Deployed Engineer",
place: str = "Garching", body: str = "The posting.", day: int = 20,
**raw: object) -> Item:
item = Item(source_id=source_id, location=f"https://jobs.example/{source_id}",
title=title, body=body, published_at=datetime(2026, 9, day, tzinfo=UTC),
raw={"company": company, "place": place, "board": "successfactors", **raw})
item.source = "ats"
return item
def dup_of(store: Store) -> dict[str, str | None]:
with sqlite3.connect(store.path) as conn:
return dict(conn.execute("SELECT uid, dup_of FROM roles"))
def test_a_board_under_several_ids_is_one_role_and_another_town_stays_apart(
store: Store) -> None:
"""SAP's repeats: three Garching rows are one kept role (the one with
the description) and two copies; New York, listed by another host, is its own role."""
garching = [role("1", body=""), role("2"), role("3", body="")]
york = role("4", place="New York, NY, US", board="greenhouse")
for item in (*garching, york):
store.upsert_role(item)
assert store.rebuild_dupes() == (1, 2, 2)
kept = garching[1].uid
assert dup_of(store) == {garching[0].uid: kept, kept: None,
garching[2].uid: kept, york.uid: None}
assert store.rebuild_dupes().changed == 0 # stable between runs
def test_a_search_copy_of_a_board_role_is_hidden_behind_the_board(store: Store) -> None:
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
board="teamtailor", body="")
search = role("search", company="Trackunit", title="Backend Engineer",
place="Aalborg, Denmark", board=None, provider="jobindex")
for item in (search, board):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store) == {board.uid: None, search.uid: board.uid}
def test_a_freehire_copy_of_a_greenhouse_role_is_hidden_behind_the_board(
store: Store, search_stubs: dict[str, object]) -> None:
"""freehire aggregates other hosts' postings; the same role it lists from
a board roledesk reads is one role, and the board's copy is the one kept
even though freehire's is the one with a description."""
board = role("board", company="Lunar", title="Head of Data", place="Copenhagen",
board="greenhouse", body="", day=22)
copy = role("copy", company="Lunar", title="Head of Data", place="Copenhagen, Denmark",
board=None, provider="freehire", source="greenhouse", day=20)
for item in (copy, board):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store) == {board.uid: None, copy.uid: board.uid}
def kept_of(store: Store, *items: Item) -> str:
"""The uid of the one of these roles that is not a copy."""
found = dup_of(store)
(kept,) = [i.uid for i in items if found[i.uid] is None]
return kept
def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
store: Store) -> None:
"""One posting per place on one host is one role that lists both places,
each with its own link; the count of copies does not show it as `x2`."""
berlin, munich = role("b", place="Berlin"), role("m", place="Munich")
for item in (berlin, munich):
store.upsert_role(item)
assert store.rebuild_dupes() == (1, 1, 1)
kept = kept_of(store, berlin, munich)
got = store.role(kept)
assert got is not None
assert sorted(p.text for p in got.places) == ["Berlin", "Munich"]
assert sorted(p.url for p in got.places if p.url) in (
["https://jobs.example/b"], ["https://jobs.example/m"])
assert got.copies == []
(row,) = store.roles_page()
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
def test_a_posting_per_office_in_two_countries_is_two_roles(store: Store) -> None:
"""OpenAI posts one role in San Francisco and in Dublin. The fold keeps
to one country (ADR 1.4), so each is a role of its own: a Europe gate
takes Dublin and rejects San Francisco, and the Place filter finds
Dublin alone."""
from app import prep
from app.models import Gate
sf = role("sf", company="OpenAI", place="San Francisco", country="US", board="ashby")
dublin = role("dub", company="OpenAI", place="Dublin, Ireland", country="IE", board="ashby")
for item in (sf, dublin):
store.upsert_role(item)
prep.run(store)
assert dup_of(store) == {sf.uid: None, dublin.uid: None}
agent = store.save_agent("A", gates=[Gate(field="country", values=["FR", "IE", "EU"])])
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {
sf.uid: [0], dublin.uid: [1]}
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {dublin.uid}
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
a = role("a", place="Berlin", body="Build payment services in Kotlin for our bank " * 5)
b = role("b", place="Munich", body="Sell cloud contracts to enterprise buyers " * 5)
c = role("c", place="Hamburg", board="greenhouse", body=a.body)
for item in (a, b, c):
store.upsert_role(item)
assert store.rebuild_dupes() == (0, 0, 0)
def own_and_copy_places(store: Store, uid: str) -> list[tuple[str, bool]]:
got = store.role(uid)
assert got is not None
return sorted((p.text, bool(p.url)) for p in got.places)
def test_a_place_copy_closing_or_its_board_off_takes_its_place_away(store: Store) -> None:
from app.jobs.sources import set_fetch
sub = store.add_subscription("ats", "successfactors:sap", label="SAP")
items = [role("b", place="Berlin"), role("m", place="Munich"), role("k", place="Kiel")]
for item in items:
store.upsert_role(item, sub)
store.rebuild_dupes()
kept = kept_of(store, *items)
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Kiel", "Munich"]
closing = next(i for i in items if i.uid != kept)
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (closing.uid,))
store.rebuild_dupes()
assert len(own_and_copy_places(store, kept)) == 2
assert store.rebuild_dupes().changed == 0
set_fetch(store, sub.id, False)
assert set(dup_of(store).values()) == {None}
assert not any(url for _, url in own_and_copy_places(store, kept))
def test_a_repull_keeps_the_copies_places(store: Store) -> None:
items = [role("b", place="Berlin"), role("m", place="Munich")]
for item in items:
store.upsert_role(item)
store.rebuild_dupes()
kept = kept_of(store, *items)
store.upsert_role(next(i for i in items if i.uid == kept))
assert [t for t, _ in own_and_copy_places(store, kept)] == ["Berlin", "Munich"]
def test_closing_the_kept_role_hands_the_group_on_and_a_closed_copy_is_cleared(
store: Store) -> None:
first, second, third = role("1", day=18), role("2", day=19), role("3", day=20)
for item in (first, second, third):
store.upsert_role(item)
store.rebuild_dupes()
assert dup_of(store)[second.uid] == first.uid
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
store.rebuild_dupes()
assert dup_of(store) == {first.uid: None, second.uid: None, third.uid: second.uid}
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-26' WHERE uid = ?", (third.uid,))
store.rebuild_dupes()
assert set(dup_of(store).values()) == {None}
def test_read_and_votes_move_to_the_kept_role(store: Store) -> None:
"""A copy read and liked before it was known as one: the kept role
becomes read and liked. An agent that voted on the kept role keeps its
own vote, and a later unread on the kept role is not undone."""
one, two = store.save_agent("One"), store.save_agent("Two")
kept, copy = role("kept", day=18), role("copy", day=19)
store.upsert_role(copy)
store.set_vote(one.id, copy.uid, 1)
store.set_vote(two.id, copy.uid, -1)
store.upsert_role(kept)
store.set_vote(two.id, kept.uid, 0)
store.rebuild_dupes()
read = store.role(copy.uid).read_at # type: ignore[union-attr]
assert store.role(kept.uid).read_at == read # type: ignore[union-attr]
assert store.votes(one.id) == {kept.uid: 1}
assert store.votes(two.id) == {kept.uid: 0, copy.uid: -1}
store.set_read(kept.uid, False)
store.rebuild_dupes()
assert store.role(kept.uid).read_at is None # type: ignore[union-attr]
def test_a_closed_kept_role_hands_its_read_and_vote_to_the_next(store: Store) -> None:
agent = store.save_agent("One")
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.rebuild_dupes()
store.set_vote(agent.id, first.uid, 1)
with sqlite3.connect(store.path) as conn:
conn.execute("UPDATE roles SET closed_at = '2026-09-25' WHERE uid = ?", (first.uid,))
# A third copy keeps the group alive once the first has closed.
store.upsert_role(role("3", day=20))
store.rebuild_dupes()
assert store.votes(agent.id) == {second.uid: 1}
assert store.role(second.uid).read_at is not None # type: ignore[union-attr]
def test_a_copy_is_left_out_of_the_feed_counts_and_model_stages(store: Store) -> None:
kept, copy, other = role("1", day=18), role("2", day=19), role("3", place="Berlin", board="greenhouse")
for item in (kept, copy, other):
store.upsert_role(item)
store.set_tags(item.uid, ["python"])
store.rebuild_dupes()
agent = store.save_agent("A", intent="engineering")
assert {r.uid for r in store.roles()} == {kept.uid, other.uid}
rows = {r.uid: r.copies for r in store.roles_page()}
assert rows == {kept.uid: 1, other.uid: 0}
assert store.count_roles() == 2
assert sum(n for _, n in store.role_facets()["seniority"]) == 2
assert store.tag_counts() == [("python", 2, 2)]
assert {c.uid for c in store.score_candidates(agent.id, [])} == {kept.uid, other.uid}
assert {r.uid for r in store.roles_to_embed("m", 10)} == {kept.uid, other.uid}
assert store.open_role_counts() == {"sap": 2}
assert [count for _, _, count, _ in store.open_companies()] == [2]
store.replace_scores(agent.id, [(u, 0.9, "", "", None, 0.9, "fit")
for u in (kept.uid, copy.uid, other.uid)])
assert {r.uid for r in store.extract_waiting(10)} == {kept.uid, other.uid}
assert {r.uid for r in store.summarise_waiting(10)} == {kept.uid, other.uid}
# The copy still opens by uid, and the kept role lists it.
assert store.role(copy.uid).dup_of == kept.uid # type: ignore[union-attr]
assert [c.uid for c in store.role(kept.uid).copies] == [copy.uid] # type: ignore[union-attr]
assert store.role(other.uid).copies == [] # type: ignore[union-attr]
def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
path = tmp_path / "v17.db"
store = Store(path)
first, second = role("1", day=18), role("2", day=19)
for item in (first, second):
store.upsert_role(item)
store.put_vectors("m", [(first.uid, np.ones(3, dtype=np.float32))])
with sqlite3.connect(path) as conn:
for table in ("role_places", "place_includes", "place_aliases", "places"):
conn.execute(f"DROP TABLE {table}")
for column in ("teaser", "seniority_model", "seniority_stated", "seniority_title"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
conn.execute("DROP TABLE company_aliases")
for column in ("sector", "stage", "size_band", "founded", "last_funding",
"last_funding_currency", "facts_source", "facts_at"):
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
conn.execute("DROP INDEX idx_roles_dup_of")
conn.execute("ALTER TABLE roles DROP COLUMN dup_of")
conn.execute("PRAGMA user_version = 17")
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
).fetchone() is not None
def test_switching_a_board_off_shows_its_roles_copies_at_once(store: Store) -> None:
"""A close outside a pull regroups too: the search's copy of a role whose
board is switched off is listed again without waiting for a pull."""
from app.jobs.sources import set_fetch
sub = store.add_subscription("ats", "teamtailor:trackunit", label="Trackunit")
board = role("board", company="Trackunit", title="Backend Engineer", place="Aalborg",
board="teamtailor")
search = role("search", company="Trackunit", title="Backend Engineer",
place="Aalborg, Denmark", board=None, provider="jobindex")
store.upsert_role(board, sub)
store.upsert_role(search)
store.rebuild_dupes()
assert dup_of(store)[search.uid] == board.uid
set_fetch(store, sub.id, False)
assert dup_of(store) == {board.uid: None, search.uid: None}
def test_a_role_in_several_cities_leads_with_the_one_nearest_the_agent(store: Store) -> None:
"""A Danish company posts one role per city. For an agent whose towns
hold Copenhagen the kept Aarhus role leads with Copenhagen, in the row
and the reader, and opens the Copenhagen copy; without an agent the
host's order stands."""
from app import prep
from app.models import Gate
items = [role(c, company="Danske", place=p, board="ashby") for c, p in (
("aar", "Aarhus"), ("bil", "Billund"), ("cph", "Copenhagen"))]
for item in items:
store.upsert_role(item)
prep.run(store)
kept = kept_of(store, *items)
agent = store.save_agent("A", gates=[
Gate(field="country", values=["FR", "DK", "EU"], mode="require"),
Gate(field="place", values=["copenhagen"], mode="any")])
got = store.role(kept, agent.id)
assert got is not None
own = next(p.text for p in got.places if not p.url)
assert own != "Copenhagen"
assert (got.lead_place, got.lead_url) == ("Copenhagen", "https://jobs.example/cph")
assert got.places[0].text == "Copenhagen"
(row,) = store.roles_page(agent=agent.id, by_score=False)
assert (row.place, row.lead_place, row.lead_url) == ("Copenhagen +2", "Copenhagen", got.lead_url)
assert row.place_texts[0] == "Copenhagen" and len(row.place_texts) == 3
plain = store.role(kept)
assert plain is not None
assert plain.lead_place == own and plain.lead_url == ""
(row,) = store.roles_page(by_score=False)
assert row.place == f"{own} +2" and row.lead_url == ""