Keep a board's stated pay as salary_text, even when it does not normalise

Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.

Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet committed 2026-09-29 15:08:45 +02:00
1 parent 706b127bc0
commit fa4e1a657f
18 files changed
+528 -57

No files matched your search

+34
View File
@@ -29,6 +29,40 @@
"publishedAt": "2026-08-28T21:20:40.792+00:00",
"jobUrl": "https://ashby.example/2",
"descriptionHtml": ""
},
{
"id": "3003",
"title": "Sales Development Rep",
"department": "Sales",
"team": "Sales",
"location": "Remote",
"isRemote": true,
"publishedAt": "2026-08-20T00:00:00Z",
"jobUrl": "https://ashby.example/3",
"descriptionHtml": "",
"compensation": {
"compensationTierSummary": "Competitive OTE, details on request",
"summaryComponents": []
}
},
{
"id": "3004",
"title": "Support Engineer (Contract)",
"department": "Support",
"team": "Support",
"location": "Remote",
"isRemote": true,
"publishedAt": "2026-08-21T00:00:00Z",
"jobUrl": "https://ashby.example/4",
"descriptionHtml": "",
"compensation": {
"summaryComponents": [
{"compensationType": "Salary", "interval": "1 HOUR", "currencyCode": "USD",
"minValue": 45, "maxValue": 60},
{"compensationType": "Bonus", "interval": "1 YEAR", "currencyCode": "USD",
"minValue": 5000, "maxValue": 10000}
]
}
}
]
}
+28
View File
@@ -40,6 +40,34 @@
"location": {"name": "Anywhere"},
"absolute_url": "data:text/html,hi",
"departments": []
},
{
"id": 1005,
"title": "Regional Sales Manager",
"location": {"name": "Remote - US"},
"absolute_url": "https://gh.example/5",
"updated_at": "2026-09-12T12:00:00-04:00",
"departments": [{"id": 2, "name": "Sales"}],
"content": "",
"pay_input_ranges": [
{"min_cents": 18000000, "max_cents": 22000000, "currency_type": "USD",
"title": "Zone 1"},
{"min_cents": 16000000, "max_cents": 20000000, "currency_type": "USD",
"title": "Zone 2"}
]
},
{
"id": 1006,
"title": "Support Specialist",
"location": {"name": "Remote"},
"absolute_url": "https://gh.example/6",
"updated_at": "2026-09-13T12:00:00-04:00",
"departments": [{"id": 3, "name": "Support"}],
"content": "",
"pay_input_ranges": [
{"min_cents": 6500000, "max_cents": 8500000, "currency_type": "",
"title": "Base salary range:"}
]
}
]
}
+10
View File
@@ -18,5 +18,15 @@
"workplaceType": "on-site",
"description": "",
"salaryRange": {"min": 25, "max": 35, "currency": "USD", "interval": "per-hour-wage"}
},
{
"id": "2003",
"text": "Support Engineer",
"hostedUrl": "https://lever.example/3",
"createdAt": 1789430400000,
"categories": {"location": "Remote", "department": "Support"},
"workplaceType": "remote",
"description": "",
"salaryRange": {"min": 80000, "max": 100000, "currency": "", "interval": "per-year-salary"}
}
]
+18
View File
@@ -41,4 +41,22 @@
<createdAt>2026-09-21T14:53:35+00:00</createdAt>
</position>
<position>
<id>6003</id>
<subcompany>Acme AG</subcompany>
<office>Munich</office>
<department>Operations</department>
<name>Office Coordinator</name>
<jobDescriptions></jobDescriptions>
<employmentType>permanent</employmentType>
<seniority>entry-level</seniority>
<schedule>full-time</schedule>
<createdAt>2026-09-05T09:00:00+00:00</createdAt>
<salaryInformation>
<min>3000</min>
<max>3600</max>
<type>monthly</type>
</salaryInformation>
</position>
</workzag-jobs>
+13
View File
@@ -27,6 +27,19 @@
"published_at": "2026-09-01 00:00:00 UTC",
"description": "",
"salary": {"min": null, "max": null, "currency": null, "period": null}
},
{
"id": 5003,
"title": "Warehouse Coordinator",
"slug": "warehouse-coordinator",
"careers_url": "https://acme.example/o/warehouse-coordinator",
"department": "Operations",
"city": "Munich",
"location": "Munich",
"remote": false,
"published_at": "2026-09-05 00:00:00 UTC",
"description": "",
"salary": {"min": 18, "max": 24, "currency": null, "period": "hour"}
}
]
}
+65 -11
View File
@@ -31,9 +31,9 @@ def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the two real ones
# The javascript: and data: rows are refused, same as the four real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed
assert first["location"] == "Berlin, Germany"
@@ -53,9 +53,22 @@ def test_greenhouse_reads_pay_input_ranges_and_skips_an_hourly_one() -> None:
(None, None, "")
def test_greenhouse_salary_text_covers_the_single_hourly_and_zoned_cases() -> None:
"""`salary_text` is written whether or not the pay normalises: a single
yearly range reads plainly, the hourly one that the columns dropped
still reads as a line, several zones are joined and labelled with the
board's own titles, and a range with no currency says so."""
rows = _rows("greenhouse")
assert rows[0]["salary_text"] == "EUR 65,000 to 85,000 per year"
assert rows[1]["salary_text"] == "USD 30 to 45 per hour"
assert rows[2]["salary_text"] == ("Zone 1: USD 180,000 to 220,000 per year; "
"Zone 2: USD 160,000 to 200,000 per year")
assert rows[3]["salary_text"] == "65,000 to 85,000 per year (no currency stated)"
def test_lever_parses_categories_and_workplace_type() -> None:
rows = _rows("lever")
assert [r["source_id"] for r in rows] == ["2001", "2002"]
assert [r["source_id"] for r in rows] == ["2001", "2002", "2003"]
first = rows[0]
assert first["department"] == "Engineering"
assert first["remote"] == "remote"
@@ -72,11 +85,18 @@ def test_lever_reads_salary_range_and_skips_an_hourly_wage() -> None:
(None, None, "")
def test_lever_salary_text_covers_the_hourly_and_no_currency_cases() -> None:
rows = _rows("lever")
assert rows[0]["salary_text"] == "EUR 90,000 to 120,000 per year"
assert rows[1]["salary_text"] == "USD 25 to 35 per hour"
assert rows[2]["salary_text"] == "80,000 to 100,000 per year (no currency stated)"
def test_ashby_parses_is_remote_and_compensation() -> None:
rows = _rows("ashby")
assert [r["source_id"] for r in rows] == ["3001", "3002"]
assert [r["source_id"] for r in rows] == ["3001", "3002", "3003", "3004"]
assert rows[0]["remote"] is False
assert rows[0]["salary"] == "70,000-90,000 EUR"
assert rows[0]["salary_text"] == "EUR 70,000 to 90,000 per year"
assert rows[1]["remote"] is True
@@ -90,6 +110,20 @@ def test_ashby_reads_summary_components_only_with_includecompensation() -> None:
(None, None, "")
def test_ashby_salary_text_falls_back_to_prose_and_labels_several_components() -> None:
"""No component with a bound at all (job 3): Ashby's own prose summary is
what is left to show. An hourly Salary component beside a yearly Bonus
one (job 4): neither normalises (`_ashby_salary` only reads a yearly or
monthly `Salary` component), but both are stated, labelled since there
is more than one."""
rows = _rows("ashby")
assert rows[2]["salary_text"] == "Competitive OTE, details on request"
assert rows[3]["salary_text"] == \
"Salary: USD 45 to 60 per hour; Bonus: USD 5,000 to 10,000 per year"
assert (rows[3]["salary_min"], rows[3]["salary_max"], rows[3]["currency"]) == \
(None, None, "")
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
rows = _rows("workable", slug="huggingface")
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
@@ -100,6 +134,7 @@ def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
assert rows[1]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is False
assert rows[0]["description"] == "" # the list endpoint carries no text
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
@@ -108,23 +143,31 @@ def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
assert rows[0]["url"] == "https://jobs.smartrecruiters.com/Acme/744000151326819"
assert rows[0]["location"] == "Berlin, Germany"
assert rows[1]["remote"] is True
assert rows[0]["salary_text"] == "" # SmartRecruiters' list endpoint states no pay at all
def test_recruitee_parses_salary_bounds_and_careers_url() -> None:
rows = _rows("recruitee")
assert [r["source_id"] for r in rows] == ["5001", "5002"]
assert [r["source_id"] for r in rows] == ["5001", "5002", "5003"]
assert rows[0]["url"] == "https://acme.example/o/player-support-agent-german-1"
assert rows[0]["salary"] == "35000-45000 EUR"
assert rows[1]["salary"] == ""
assert rows[0]["salary_text"] == "EUR 35,000 to 45,000 per year"
assert rows[1]["salary_text"] == ""
assert (rows[0]["salary_min"], rows[0]["salary_max"], rows[0]["currency"]) == \
(35000, 45000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
def test_recruitee_salary_text_states_an_hourly_rate_with_no_currency() -> None:
rows = _rows("recruitee")
assert rows[2]["salary_text"] == "18 to 24 per hour (no currency stated)"
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
(None, None, "")
def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
rows = _rows("personio", slug="acme")
assert [r["source_id"] for r in rows] == ["6001", "6002"]
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
assert rows[0]["description"] == "Grow our pipeline."
assert rows[0]["remote"] is True # "Remote - Germany"
@@ -137,6 +180,17 @@ def test_personio_reads_salary_information_when_present() -> None:
(42000, 55000, "EUR")
assert (rows[1]["salary_min"], rows[1]["salary_max"], rows[1]["currency"]) == \
(None, None, "")
assert rows[0]["salary_text"] == "EUR 42,000 to 55,000 per year"
assert rows[1]["salary_text"] == ""
def test_personio_salary_text_states_a_monthly_rate_with_no_currency() -> None:
rows = _rows("personio", slug="acme")
assert rows[2]["salary_text"] == "3,000 to 3,600 per month (no currency stated)"
# No currency: dropped from the columns like any other host's, even
# though the interval itself (monthly) is one _yearly_amount knows.
assert (rows[2]["salary_min"], rows[2]["salary_max"], rows[2]["currency"]) == \
(None, None, "")
# ------------------------------------------------------------ salary math
@@ -265,7 +319,7 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true&pay_transparency=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
]
assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
assert complete # a host that does not page answers with the whole board
@@ -434,7 +488,7 @@ def test_an_answer_clears_the_failure_and_stamps_completeness(
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 2
assert len(list(source.fetch())) == 4
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""
+2 -2
View File
@@ -36,7 +36,7 @@ def _item(board: str, slug: str, job_id: str) -> Item:
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "place": "Berlin", "department": "",
"function": "engineering", "remote": None, "salary": "",
"function": "engineering", "remote": None, "salary_text": "",
"board": board, "slug": slug})
item.source = "ats"
return item
@@ -170,7 +170,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary": ""}
"description": "", "remote": None, "salary_text": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
+58 -2
View File
@@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 11
assert LATEST == 12
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -92,7 +92,7 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
with sqlite3.connect(old) as conn:
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
for column in ("permit", "languages", "extract_tries", "summary", "summarised_at",
"article_text", "article_at", "read_at", "posted_at"):
"article_text", "article_at", "read_at", "posted_at", "salary_text"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("intent_key", "intent_vec", "limits"): # v3, v8
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
@@ -119,6 +119,7 @@ def test_a_v8_file_gains_ats_slug_and_keeps_its_companies(tmp_path: Path) -> Non
conn.execute("ALTER TABLE companies DROP COLUMN ats_slug")
conn.execute("DROP INDEX idx_roles_posted") # v10, which follows
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12, which follows
conn.execute("PRAGMA user_version = 8")
migrated = Store(path)
@@ -136,6 +137,7 @@ def _downgrade_to_v4(path: Path) -> None:
conn.execute("ALTER TABLE roles DROP COLUMN read_at")
conn.execute("DROP INDEX idx_roles_posted") # v9: indexes posted_at
conn.execute("ALTER TABLE roles DROP COLUMN posted_at") # v9
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12
conn.execute("DROP TABLE role_tags")
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
@@ -245,6 +247,7 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
with sqlite3.connect(path) as conn:
conn.execute("DROP INDEX idx_roles_posted")
conn.execute("ALTER TABLE roles DROP COLUMN posted_at")
conn.execute("ALTER TABLE roles DROP COLUMN salary_text") # v12
conn.execute("PRAGMA user_version = 9")
migrated = Store(path)
@@ -342,6 +345,40 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None:
"""`salary_text` behaves like `title` or `published_at`, not like the
rest of `_RAW_TEXT`: a board's own pay line replaces what was stored on
every pull, and a pull that states no pay at all clears it rather than
leaving a stale line behind — the opposite of the keep-filled rule
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year"))
role = store.role(posting().uid)
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
# The board stops stating pay: the stored line is cleared, not kept.
store.upsert_role(posting())
role = store.role(posting().uid)
assert role is not None and role.salary_text == ""
def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) -> None:
"""Not a stored column (docs/decisions.md): 'normalised' needs only
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
'stated' is `salary_text` alone; neither is ''."""
store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year",
salary_min=70000, salary_max=90000, currency="EUR"))
store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("3"))
by_id = {r.source_id: r for r in store.roles()}
assert by_id["1"].salary_state == "normalised"
assert by_id["2"].salary_state == "stated"
assert by_id["3"].salary_state == ""
def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
store: Store, clock: Clock) -> None:
"""A board's first pull can carry a stale, too-recent published_at (a
@@ -504,6 +541,25 @@ def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) ->
assert [r.source_id for r in store.roles(days=30)] == ["2", "1"]
def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> None:
"""`pay` filters in SQL, the same as the other filters `_roles_from`
builds, so it narrows what a `limit` is taken from rather than the page
already built (docs/decisions.md, "filtering before LIMIT")."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
clock.tick()
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
clock.tick()
store.upsert_role(posting("none"))
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
assert {r.source_id for r in store.roles()} == {"normalised", "stated", "none"}
assert store.count_roles(pay="stated") == 2
with pytest.raises(ValueError, match="pay is"):
store.roles(pay="bogus")
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
agent = store.save_agent("A")
store.add_company("Acme", key="acme")
+24
View File
@@ -88,6 +88,30 @@ def test_list_roles_filters_by_query_place_and_function(
assert [r["source_id"] for r in res.json()["items"]] == ["3"]
def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) -> None:
"""`salary_text` and `salary_state` (app/models.py) are on every role the
API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before
`limit` like the other filters."""
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
items = {r["source_id"]: r for r in client.get("/api/roles").json()["items"]}
assert items["normalised"]["salary_text"] == "EUR 70,000 to 90,000 per year"
assert items["normalised"]["salary_state"] == "normalised"
assert items["stated"]["salary_state"] == "stated"
assert items["none"]["salary_state"] == ""
res = client.get("/api/roles", params={"pay": "stated"})
assert {r["source_id"] for r in res.json()["items"]} == {"normalised", "stated"}
res = client.get("/api/roles", params={"pay": "normalised"})
assert [r["source_id"] for r in res.json()["items"]] == ["normalised"]
res = client.get("/api/roles", params={"pay": "stated_only"})
assert [r["source_id"] for r in res.json()["items"]] == ["stated"]
assert client.get("/api/roles", params={"pay": "bogus"}).status_code == 422
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
for i in range(3):
store.upsert_role(posting(str(i)))