Merge branch 'worktree-agent-afad54f12aef2cccf'
# Conflicts: # app/storage/schema.py # app/storage/store.py # tests/test_storage.py
This commit is contained in:
commit
706b127bc0
10 files changed
+459
-37
No files matched your search
+199
-18
@@ -62,11 +62,13 @@ MAX_BYTES = 96 << 20
|
||||
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
||||
#: recruitee and personio put the slug in the hostname instead of the path.
|
||||
_BOARDS: dict[str, tuple[str, str]] = {
|
||||
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs?content=true",
|
||||
"greenhouse": ("https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
|
||||
"?content=true&pay_transparency=true",
|
||||
"https://job-boards.greenhouse.io/{slug}"),
|
||||
"lever": ("https://api.lever.co/v0/postings/{slug}?mode=json",
|
||||
"https://jobs.lever.co/{slug}"),
|
||||
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}",
|
||||
"ashby": ("https://api.ashbyhq.com/posting-api/job-board/{slug}"
|
||||
"?includeCompensation=true",
|
||||
"https://jobs.ashbyhq.com/{slug}"),
|
||||
"workable": ("https://apply.workable.com/api/v3/accounts/{slug}/jobs",
|
||||
"https://apply.workable.com/{slug}"),
|
||||
@@ -195,6 +197,88 @@ def _parse_dt(value: Any) -> datetime | None:
|
||||
Row = dict[str, Any]
|
||||
|
||||
|
||||
# ---------------------------------------------------------- structured pay
|
||||
#
|
||||
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
||||
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
||||
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
||||
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
||||
# structured pay on their list endpoint; Workable and SmartRecruiters do not
|
||||
# (checked live, 2026-09-29), so their parsers are untouched and `row.get`
|
||||
# below reads a plain '' or None for them.
|
||||
|
||||
|
||||
def _num(value: Any) -> float | None:
|
||||
"""A positive number from whatever a board sends (int, float, or a
|
||||
numeral string), or None for anything else, including zero and a
|
||||
negative — a board's placeholder 0 (an unfilled field, not a real
|
||||
figure) must not become a salary floor of zero."""
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if number > 0 else None
|
||||
|
||||
|
||||
def _yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
||||
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
||||
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
||||
None, "")` when there is no currency, no bound, or `interval` is not one
|
||||
of `factors`' own keys — which is how an hourly or a daily rate is left
|
||||
out rather than annualised into a number nobody stated. `factors` maps
|
||||
the board's own interval spelling to the multiplier that makes it
|
||||
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
||||
code = str(currency or "").strip().upper()
|
||||
factor = factors.get(str(interval or "").strip())
|
||||
if not code or not factor:
|
||||
return None, None, ""
|
||||
lo, hi = _num(min_value), _num(max_value)
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo * factor) if lo is not None else None,
|
||||
int(hi * factor) if hi is not None else None, code)
|
||||
|
||||
|
||||
def _greenhouse_salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
|
||||
on the feed URL, see `_BOARDS`), which the list endpoint already carries
|
||||
in the same request — no per-role fetch. Only when there is exactly one
|
||||
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
|
||||
one per office) gives no way to tell which office the role in hand is
|
||||
at, and picking one would misrepresent the columns. Greenhouse's own
|
||||
ranges are cents, and yearly by convention except an hourly or a daily
|
||||
one, named as such in `title` with no separate field to check — those
|
||||
are skipped rather than annualised into a number nobody stated."""
|
||||
if len(ranges) != 1:
|
||||
return None, None, ""
|
||||
entry = ranges[0]
|
||||
title = str(entry.get("title") or "").lower()
|
||||
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
|
||||
return None, None, ""
|
||||
code = str(entry.get("currency_type") or "").strip().upper()
|
||||
if not code:
|
||||
return None, None, ""
|
||||
lo, hi = _num(entry.get("min_cents")), _num(entry.get("max_cents"))
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo / 100) if lo is not None else None,
|
||||
int(hi / 100) if hi is not None else None, code)
|
||||
|
||||
|
||||
def _ashby_salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `compensation.summaryComponents` (needs
|
||||
`?includeCompensation=true` on the feed URL, see `_BOARDS`): the first
|
||||
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
|
||||
an hourly one is left out rather than annualised."""
|
||||
for component in components:
|
||||
if component.get("compensationType") != "Salary":
|
||||
continue
|
||||
return _yearly_amount(component.get("minValue"), component.get("maxValue"),
|
||||
component.get("currencyCode"), component.get("interval"),
|
||||
{"1 YEAR": 1, "1 MONTH": 12})
|
||||
return None, None, ""
|
||||
|
||||
|
||||
def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
@@ -208,6 +292,7 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
salary_min, salary_max, currency = _greenhouse_salary(j.get("pay_input_ranges") or [])
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -218,6 +303,9 @@ def _parse_greenhouse(body: bytes) -> list[Row]:
|
||||
"description": strip_html(content),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary": "",
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -234,6 +322,10 @@ def _parse_lever(body: bytes) -> list[Row]:
|
||||
continue
|
||||
categories = j.get("categories") or {}
|
||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||
salary_range = j.get("salaryRange") or {}
|
||||
salary_min, salary_max, currency = _yearly_amount(
|
||||
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
|
||||
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -244,6 +336,9 @@ def _parse_lever(body: bytes) -> list[Row]:
|
||||
"description": strip_html(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary": "",
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -260,6 +355,7 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
compensation = j.get("compensation") or {}
|
||||
salary = (compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
salary_min, salary_max, currency = _ashby_salary(compensation.get("summaryComponents") or [])
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -270,6 +366,9 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("isRemote"),
|
||||
"salary": salary,
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -361,6 +460,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
|
||||
if isinstance(salary, dict) and (salary.get("min") or salary.get("max")):
|
||||
bounds = "-".join(str(v) for v in (salary.get("min"), salary.get("max")) if v)
|
||||
salary_text = f"{bounds} {salary.get('currency') or ''}".strip()
|
||||
salary_min, salary_max, currency = _yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
salary.get("period"), {"year": 1, "month": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -371,6 +473,9 @@ def _parse_recruitee(body: bytes) -> list[Row]:
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"remote": j.get("remote"),
|
||||
"salary": salary_text,
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -388,6 +493,14 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
||||
continue
|
||||
description = " ".join(
|
||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||
salary_info = position.find("salaryInformation")
|
||||
salary_min = salary_max = None
|
||||
currency = ""
|
||||
if salary_info is not None:
|
||||
salary_min, salary_max, currency = _yearly_amount(
|
||||
salary_info.findtext("min"), salary_info.findtext("max"),
|
||||
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
||||
{"yearly": 1, "monthly": 12})
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
@@ -398,6 +511,9 @@ def _parse_personio(body: bytes, slug: str) -> list[Row]:
|
||||
"description": strip_html(description),
|
||||
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
||||
"salary": "",
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
|
||||
@@ -417,14 +533,27 @@ _PARSERS: dict[str, Callable[..., list[Row]]] = {
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
|
||||
#: The fixed set of functions, and the keyword rules that map a department
|
||||
#: (preferred) or a title to one of them. Ported verbatim, in order, from
|
||||
#: `cmd/seedgen/main.go`'s `rules` — first match wins, and "product" is
|
||||
#: genuinely listed twice: the first, narrower entry is checked before
|
||||
#: The fixed set of functions, and the keyword rules that map a title
|
||||
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
||||
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
||||
#: twice: the first, narrower entry is checked before
|
||||
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
||||
#: matches the bare word "product", is checked only after all of those, so a
|
||||
#: title like "Data Product Manager" keeps classifying as data/product by the
|
||||
#: same rule order the Go tool used.
|
||||
#:
|
||||
#: Not a straight port any more: the Go tool matched substrings, which let
|
||||
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
||||
#: "Italia" — a European department like "Commercial Italy" filed as
|
||||
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
||||
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
||||
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
||||
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
||||
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
||||
#: meant to also catch "management", "engineering", "recruiting", and a
|
||||
#: trailing boundary on those would stop them matching the plural and
|
||||
#: gerund forms real postings use ("Software Developers", "Account
|
||||
#: Managers").
|
||||
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
||||
"customer success/support", "recruiting/people", "finance/legal/ops")
|
||||
|
||||
@@ -433,7 +562,7 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
"program manag", "technical program")),
|
||||
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
||||
"people team", "people operations", "human resources",
|
||||
" hr ", "hr ", "hrbp")),
|
||||
"hr", "hrbp")),
|
||||
("design", ("design", "ux", "user experience", "creative")),
|
||||
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
||||
"seo", "social media", "demand gen", "events")),
|
||||
@@ -446,8 +575,17 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("sales", ("sales", "account executive", "account manager", "account director",
|
||||
"business development", "bdr", "sdr", "revenue", "partnership",
|
||||
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
||||
"pre-sales", "presales", "channel")),
|
||||
("data", ("data", "analytics", "machine learning", "ml ", " ml",
|
||||
"pre-sales", "presales", "channel",
|
||||
# "Commercial" is what French, Italian and Spanish org charts
|
||||
# call the sales function ("Commercial Italy" is a sales
|
||||
# department; see the docstring above), and the rest are the
|
||||
# sales words of the seed's other languages that "sales"
|
||||
# itself does not already cover (German, Dutch, Danish,
|
||||
# Norwegian, Swedish, French) — a modest set, not a full port;
|
||||
# `extract` is the rest.
|
||||
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
||||
"försäljning", "vente")),
|
||||
("data", ("data", "analytics", "machine learning", "ml",
|
||||
"applied science", "research scientist", "statistic")),
|
||||
("product", ("product manag", "product management", "program manag",
|
||||
"product operations", "technical program", "product")),
|
||||
@@ -460,28 +598,64 @@ _RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
"trust & safety", "trust and safety", "policy")),
|
||||
("engineering", ("engineer", "software", "developer", "infrastructure",
|
||||
"platform", "devops", "sre", "site reliability", "security",
|
||||
"information technology", " it", "it ", "technology",
|
||||
"information technology", "it", "technology",
|
||||
"technical staff", "hardware", "qa", "quality assurance",
|
||||
"architect", "research", "science", "scientist", "systems",
|
||||
"network", "cloud", "mobile", "frontend", "backend",
|
||||
"full stack", "fullstack", "web")),
|
||||
"full stack", "fullstack", "web",
|
||||
# The developer/engineer words of the seed's other
|
||||
# languages, same reasoning as sales' above.
|
||||
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
||||
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
||||
"développeur", "developpeur", "ingénieur")),
|
||||
)
|
||||
|
||||
#: Keywords that must match a whole word, not just its start: each one is
|
||||
#: itself the first letters of an unrelated word that would otherwise false-
|
||||
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
||||
#: not listed here keeps the old prefix behaviour on purpose — see the
|
||||
#: `_RULES` docstring above.
|
||||
_WHOLE_WORD = frozenset({
|
||||
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
||||
"sre", "bdr", "sdr", "gtm",
|
||||
})
|
||||
|
||||
|
||||
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
||||
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
||||
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
||||
|
||||
|
||||
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
||||
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
||||
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
||||
for function, keywords in _RULES)
|
||||
|
||||
|
||||
def _classify_text(text: str) -> str:
|
||||
padded = f" {text.strip().lower()} "
|
||||
if not padded.strip():
|
||||
stripped = (text or "").strip()
|
||||
if not stripped:
|
||||
return ""
|
||||
for function, keywords in _RULES:
|
||||
if any(keyword in padded for keyword in keywords):
|
||||
for function, patterns in _COMPILED_RULES:
|
||||
if any(pattern.search(stripped) for pattern in patterns):
|
||||
return function
|
||||
return ""
|
||||
|
||||
|
||||
def classify(department: str, title: str) -> str:
|
||||
"""A department (preferred) or a title, to one of `FUNCTIONS`, or
|
||||
`"other"`. Port of seedgen's `classify`/`classifyText`."""
|
||||
return _classify_text(department or "") or _classify_text(title or "") or "other"
|
||||
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
||||
when neither names one — `''` and not `"other"`, which would claim a
|
||||
knowledge the classifier does not have (`extract` may still fill it from
|
||||
reading the posting; see `Store.set_extracted`).
|
||||
|
||||
Title before department: a department is often a generic umbrella
|
||||
("Operations", "Engineering & Product") that says less about the role
|
||||
than its title does. A hand-checked sample of 60 roles where the two
|
||||
orders disagreed (docs/decisions.md) found title-first right about twice
|
||||
as often as department-first, mostly because a specific title beats a
|
||||
broad department name, so title now wins; department is the fallback
|
||||
when the title matches nothing."""
|
||||
return _classify_text(title or "") or _classify_text(department or "") or ""
|
||||
|
||||
|
||||
def is_known_target(target: str) -> bool:
|
||||
@@ -705,6 +879,13 @@ class ATSSource(Source):
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
# Only Ashby, Greenhouse, Lever, Recruitee and
|
||||
# Personio's parsers set these (see "structured
|
||||
# pay" above); the rest leave them at row.get's
|
||||
# default of no figure.
|
||||
"salary_min": row.get("salary_min"),
|
||||
"salary_max": row.get("salary_max"),
|
||||
"currency": row.get("currency", ""),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user