Keep a board's stated pay as salary_text, even when it does not normalise
Every ats parser now writes one human-readable pay line whether or not it
converts to yearly salary_min/salary_max/currency: hourly and daily rates,
Greenhouse's several pay zones, and a range with no currency all survive as
text instead of being dropped outright. Stored in roles.salary_text
(migration 12), replaced whole on every pull like title, so a board that
stops stating pay clears it rather than leaving a stale line. salary_state
is derived, not stored ('normalised', 'stated', or ''), and is what the
feed's new Pay filter and badge read; gates and scoring are unchanged, still
reading only the normalised columns.
Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
1 parent
706b127bc0
commit
fa4e1a657f
18 files changed
+528
-57
No files matched your search
+156
-19
@@ -279,6 +279,144 @@ def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | N
|
||||
return None, None, ""
|
||||
|
||||
|
||||
# ------------------------------------------------------------- salary_text
|
||||
#
|
||||
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
||||
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
||||
# range with no currency are all dropped from those columns rather than
|
||||
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
||||
# information survives — one human-readable line per parser, written
|
||||
# whether or not the pay conformed, so a person can read it and do their own
|
||||
# sums and a stricter analysis can find the roles the columns left out
|
||||
# (`Role.salary_state`, `app/models.py`).
|
||||
|
||||
#: Formats a number the way a salary line writes it: no decimals for a whole
|
||||
#: figure (the normal case for a yearly or monthly one), two for anything
|
||||
#: else (an hourly or daily rate quoted with cents).
|
||||
def _fmt_amount(value: float) -> str:
|
||||
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
||||
|
||||
|
||||
def _bounds_words(lo: float | None, hi: float | None) -> str:
|
||||
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
||||
bound stated, or "" with neither."""
|
||||
if lo is not None and hi is not None:
|
||||
return _fmt_amount(lo) if lo == hi else f"{_fmt_amount(lo)} to {_fmt_amount(hi)}"
|
||||
if lo is not None:
|
||||
return f"from {_fmt_amount(lo)}"
|
||||
if hi is not None:
|
||||
return f"up to {_fmt_amount(hi)}"
|
||||
return ""
|
||||
|
||||
|
||||
def _pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
||||
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
||||
per year" — or "" when there is no bound to report at all. A currency
|
||||
that was not stated does not drop the line, unlike the normalised
|
||||
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
||||
currency stated)", so the figure is not lost along with the currency."""
|
||||
bounds = _bounds_words(lo, hi)
|
||||
if not bounds:
|
||||
return ""
|
||||
tail = f" per {period}" if period else ""
|
||||
code = str(currency or "").strip().upper()
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
|
||||
|
||||
def _greenhouse_salary_text(ranges: list[dict[str, Any]]) -> str:
|
||||
"""One line per `pay_input_ranges` entry, joined with "; " — every range
|
||||
Greenhouse states, not only the single one `_greenhouse_salary` accepts
|
||||
for the columns. With more than one, each is labelled with the board's
|
||||
own `title` ("Zone 1 Pay Range: ..."), since that is the only way to
|
||||
tell the ranges apart; with just the one, the label adds nothing and is
|
||||
left off. The hour/day check is the same one `_greenhouse_salary` uses
|
||||
to keep those out of the yearly columns; here it only picks the word
|
||||
that follows "per"."""
|
||||
multi = len(ranges) > 1
|
||||
lines = []
|
||||
for entry in ranges:
|
||||
title = str(entry.get("title") or "").strip()
|
||||
low_title = title.lower()
|
||||
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
|
||||
else "day" if "day" in low_title else "year")
|
||||
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
|
||||
line = _pay_line(lo / 100 if lo is not None else None,
|
||||
hi / 100 if hi is not None else None,
|
||||
entry.get("currency_type"), period)
|
||||
if not line:
|
||||
continue
|
||||
if multi:
|
||||
label = title.rstrip(":").strip() or "Range"
|
||||
line = f"{label}: {line}"
|
||||
lines.append(line)
|
||||
return "; ".join(lines)
|
||||
|
||||
|
||||
def _lever_salary_text(salary_range: dict[str, Any]) -> str:
|
||||
"""One line from `salaryRange`, whatever its `interval` — including an
|
||||
hourly or daily one, which `_yearly_amount` drops from the columns.
|
||||
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
|
||||
"per-hour-wage"); the middle word is the period."""
|
||||
interval = str(salary_range.get("interval") or "")
|
||||
parts = interval.split("-")
|
||||
period = parts[1] if len(parts) >= 3 else ""
|
||||
return _pay_line(_num(salary_range.get("min")), _num(salary_range.get("max")),
|
||||
salary_range.get("currency"), period)
|
||||
|
||||
|
||||
def _ashby_salary_text(compensation: dict[str, Any]) -> str:
|
||||
"""One line per `summaryComponents` entry that states a bound, labelled
|
||||
by the board's own `compensationType` when more than one does (a salary
|
||||
range beside a bonus range, say); with just the one, the label is left
|
||||
off, the same as `_greenhouse_salary_text`. `interval` is "<n>
|
||||
<PERIOD>" ("1 YEAR", "1 HOUR"); the last word, lowered, is the period.
|
||||
With nothing to build a line from, Ashby's own prose summary
|
||||
(`compensationTierSummary`, else `scrapeableCompensationSalarySummary`)
|
||||
is what is left to show."""
|
||||
components = compensation.get("summaryComponents") or []
|
||||
stated = [c for c in components if _num(c.get("minValue")) is not None
|
||||
or _num(c.get("maxValue")) is not None]
|
||||
lines = []
|
||||
for component in stated:
|
||||
period = str(component.get("interval") or "").split(" ")[-1].lower()
|
||||
line = _pay_line(_num(component.get("minValue")), _num(component.get("maxValue")),
|
||||
component.get("currencyCode"), period)
|
||||
if not line:
|
||||
continue
|
||||
if len(stated) > 1:
|
||||
label = str(component.get("compensationType") or "").strip()
|
||||
line = f"{label}: {line}" if label else line
|
||||
lines.append(line)
|
||||
if lines:
|
||||
return "; ".join(lines)
|
||||
return str(compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
|
||||
|
||||
def _recruitee_salary_text(salary: dict[str, Any]) -> str:
|
||||
"""One line from Recruitee's `salary` object; `period` is already a
|
||||
plain word ("year", "month")."""
|
||||
period = str(salary.get("period") or "").strip().lower()
|
||||
return _pay_line(_num(salary.get("min")), _num(salary.get("max")),
|
||||
salary.get("currency"), period)
|
||||
|
||||
|
||||
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
||||
#: `_pay_line` wants ("yearly" -> "year"). Anything else is passed through.
|
||||
_PERSONIO_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
|
||||
"daily": "day", "weekly": "week"}
|
||||
|
||||
|
||||
def _personio_salary_text(salary_info: ET.Element) -> str:
|
||||
"""One line from a `<salaryInformation>` element, whatever its `type`,
|
||||
including an hourly or daily one that `_yearly_amount` drops from the
|
||||
columns."""
|
||||
period_raw = (salary_info.findtext("type") or "").strip().lower()
|
||||
period = _PERSONIO_PERIODS.get(period_raw, period_raw)
|
||||
return _pay_line(_num(salary_info.findtext("min")), _num(salary_info.findtext("max")),
|
||||
salary_info.findtext("currencyCode"), period)
|
||||
|
||||
|
||||
def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
@@ -292,7 +430,8 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or [])
|
||||
ranges = j.get("pay_input_ranges") or []
|
||||
salary_min, salary_max, currency = _greenhouse_salary(ranges)
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -302,7 +441,7 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
"department": (departments[0].get("name") if departments else "") or "",
|
||||
"description": strip_html(content),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary": "",
|
||||
"salary_text": _greenhouse_salary_text(ranges),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
@@ -335,7 +474,7 @@ def _parse_lever(body: bytes) -> list[Row]:
|
||||
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary": "",
|
||||
"salary_text": _lever_salary_text(salary_range),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
@@ -353,8 +492,6 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
continue
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary = (compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
@@ -365,7 +502,7 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("isRemote"),
|
||||
"salary": salary,
|
||||
"salary_text": _ashby_salary_text(compensation),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
@@ -408,7 +545,7 @@ def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"department": dept.strip(),
|
||||
"description": "",
|
||||
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
||||
"salary": "",
|
||||
"salary_text": "", # Workable's list endpoint states no pay at all
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -442,7 +579,7 @@ def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
||||
"description": "",
|
||||
"remote": location.get("remote"),
|
||||
"salary": "",
|
||||
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -455,11 +592,7 @@ def _parse_recruitee(body: bytes) -> list[Row]:
|
||||
url = (j.get("careers_url") or "").strip()
|
||||
if not title or not _is_web_url(url):
|
||||
continue
|
||||
salary = j.get("salary") or {}
|
||||
salary_text = ""
|
||||
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
|
||||
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
|
||||
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
|
||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||
salary_min, salary_max, currency = _yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
salary.get("period"), {"year": 1, "month": 12})
|
||||
@@ -472,7 +605,7 @@ def _parse_recruitee(body: bytes) -> list[Row]:
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"remote": j.get("remote"),
|
||||
"salary": salary_text,
|
||||
"salary_text": _recruitee_salary_text(salary),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
@@ -496,11 +629,13 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
||||
salary_info = position.find("salaryInformation")
|
||||
salary_min = salary_max = None
|
||||
currency = ""
|
||||
salary_text = ""
|
||||
if salary_info is not None:
|
||||
salary_min, salary_max, currency = _yearly_amount(
|
||||
salary_info.findtext("min"), salary_info.findtext("max"),
|
||||
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
||||
{"yearly": 1, "monthly": 12})
|
||||
salary_text = _personio_salary_text(salary_info)
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
@@ -510,7 +645,7 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
||||
"salary": "",
|
||||
"salary_text": salary_text,
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
@@ -878,11 +1013,13 @@ class ATSSource(Source):
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
# Only Ashby, Greenhouse, Lever, Recruitee and
|
||||
# Personio's parsers set these (see "structured
|
||||
# pay" above); the rest leave them at row.get's
|
||||
# Every parser sets salary_text, empty or not (see
|
||||
# "salary_text" above); only Ashby, Greenhouse,
|
||||
# Lever, Recruitee and Personio's also set the
|
||||
# normalised figures (see "structured pay"
|
||||
# above), so the rest leave them at row.get's
|
||||
# default of no figure.
|
||||
"salary_text": row.get("salary_text", ""),
|
||||
"salary_min": row.get("salary_min"),
|
||||
"salary_max": row.get("salary_max"),
|
||||
"currency": row.get("currency", ""),
|
||||
|
||||
Reference in new issue
Block a user