From c497fe965d515b35231b05f507869fb1c8d3ed12 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 21:40:12 +0200 Subject: [PATCH] Keep a description's headings, bullets and paragraphs, and draw them in the reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 --- app/models.py | 8 +- app/static/app.css | 3 + app/static/roles.js | 25 ++++-- docs/decisions.md | 2 + plugins/readers/__init__.py | 2 +- plugins/readers/_shared.py | 122 +++++++++++++++++++++++++++++ plugins/readers/ashby.py | 5 +- plugins/readers/bamboohr.py | 6 +- plugins/readers/deel.py | 6 +- plugins/readers/eightfold.py | 7 +- plugins/readers/gem.py | 7 +- plugins/readers/greenhouse.py | 12 +-- plugins/readers/hibob.py | 9 +-- plugins/readers/hrmanager.py | 6 +- plugins/readers/jibe.py | 12 +-- plugins/readers/jobylon.py | 4 +- plugins/readers/lever.py | 5 +- plugins/readers/oracle.py | 12 ++- plugins/readers/personio.py | 5 +- plugins/readers/phenom.py | 6 +- plugins/readers/pinpoint.py | 23 ++---- plugins/readers/platsbanken.py | 4 +- plugins/readers/recruitee.py | 5 +- plugins/readers/rippling.py | 6 +- plugins/readers/smartrecruiters.py | 13 ++- plugins/readers/successfactors.py | 4 +- plugins/readers/teamtailor.py | 6 +- plugins/readers/workable.py | 11 ++- plugins/readers/workday.py | 7 +- tests/test_ats.py | 2 +- tests/test_ats_bamboohr.py | 2 +- tests/test_ats_deel.py | 2 +- tests/test_ats_eightfold.py | 2 +- tests/test_ats_gem.py | 2 +- tests/test_ats_hrmanager.py | 2 +- tests/test_ats_jibe.py | 2 +- tests/test_ats_oracle.py | 2 +- tests/test_ats_phenom.py | 6 +- tests/test_ats_pinpoint.py | 10 +-- tests/test_ats_rippling.py | 4 +- tests/test_ats_smartrecruiters.py | 2 +- tests/test_ats_successfactors.py | 10 +-- tests/test_ats_workable.py | 9 +-- tests/test_readers_text.py | 47 +++++++++++ 44 files changed, 300 insertions(+), 147 deletions(-) create mode 100644 tests/test_readers_text.py diff --git a/app/models.py b/app/models.py index f9fb696..c94de92 100644 --- a/app/models.py +++ b/app/models.py @@ -1020,14 +1020,18 @@ class ScoredRole(Role): #: carries more of a role's text than a row draws. EXCERPT_LIMIT = 200 +#: The marks `html_text` starts a heading and a list item with. +_LINE_MARK = re.compile("^(## |\u2022 )", re.M) + def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str: """`text` (a role's summary, or the opening of its description) with its whitespace collapsed, cut to `limit` characters at a word boundary with an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a `SUBSTR` of the source column rather than the whole of it, so the feed's - list never carries what only the reader needs.""" - collapsed = " ".join((text or "").split()) + list never carries what only the reader needs. A description's heading + and bullet marks (`plugins.readers._shared.html_text`) are left out.""" + collapsed = " ".join(_LINE_MARK.sub("", text or "").split()) if len(collapsed) <= limit: return collapsed cut = collapsed[:limit] diff --git a/app/static/app.css b/app/static/app.css index 9c83a62..e5e5e10 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -929,6 +929,9 @@ body[data-reader="open"] .company-strip { display: none; } margin-top: 1.3rem; font-size: 1.06rem; line-height: 1.7; } .prose p { margin: 0 0 1.05rem; } +/* A description's headings and bullets (roles.js `paragraphs`), newsdesk's rules. */ +.prose h2, .prose h3, .prose h4 { line-height: 1.3; margin: 1.4rem 0 .7rem; } +.prose ul, .prose ol { padding-left: 1.5rem; } .reader-out { margin-top: 1.5rem; padding-top: 1rem; border-top: 1px solid var(--line); } .reader-note a { color: var(--accent); } .reader-copy { margin-top: .8rem; } diff --git a/app/static/roles.js b/app/static/roles.js index f2da6e7..aa77dc8 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -861,13 +861,17 @@ return role.excerpt || ""; } + /* The heading and bullet marks of a description line, see `paragraphs`. */ + const LINE_MARK = /^(## |\u2022 )/gm; + /* The same cut, run here instead: for the one row `merge` refreshes after a fetch or a summarise, off the full role that call answers with, which is the only time this page ever sees a role's `summary` or `description` directly. */ function excerptOf(role) { /* The server's order (`_EXCERPT_SOURCE`): summary, description, teaser. */ - const text = (role.summary || role.description || role.teaser || "").replace(/\s+/g, " ").trim(); + const text = (role.summary || role.description || role.teaser || "") + .replace(LINE_MARK, "").replace(/\s+/g, " ").trim(); if (text.length <= 200) return text; const cut = text.slice(0, 200); const space = cut.lastIndexOf(" "); @@ -1291,11 +1295,22 @@ try { return new URL(url).host.replace(/^www\./, ""); } catch (e) { return ""; } } - /* Plain text in paragraphs: a fetched page keeps its line breaks, and a - board's description is one long line, which stays one paragraph. */ + /* A description is a line per paragraph (`plugins.readers._shared.html_text`): + "## " starts a heading and "• " a list item, which is how a board's HTML + keeps its sections and bullets (`LINE_MARK`). A fetched page keeps its + line breaks, and an older description is one long line, which stays one + paragraph. */ function paragraphs(text) { - return text.split(/\n+/).map((p) => p.trim()).filter(Boolean) - .map((p) => `

${esc(p)}

`).join(""); + let html = ""; + let list = false; + for (const line of text.split(/\n+/).map((p) => p.trim()).filter(Boolean)) { + const item = line.startsWith("\u2022 "); + if (item !== list) { html += item ? ""; list = item; } + html += item ? `
  • ${esc(line.slice(2))}
  • ` + : line.startsWith("## ") ? `

    ${esc(line.slice(3))}

    ` + : `

    ${esc(line)}

    `; + } + return list ? `${html}` : html; } function renderReader({ loading = false, error = "" } = {}) { diff --git a/docs/decisions.md b/docs/decisions.md index 03006f2..fa98cc0 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -130,6 +130,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A list teaser is kept.** A short description in a listing (Phenom, Oracle, Jobylon, Jobindex, Adzuna) is `roles.teaser`, shown and embedded until the full text arrives, when the vector is dropped and the text replaces it. +**A description keeps its shape as marked lines.** `_shared.html_text` turns a host's HTML into a line per paragraph, `• ` before a list item and `## ` before a heading (an `

    `..`

    `, or a paragraph of at most 80 characters set wholly in bold). `strip_html` had flattened a posting into one line, so sections and bullets ran together in the reader. Plain text with marks, not sanitised HTML, is stored because summarise, extract and embed read the same column, and the reader pane escapes each line before drawing it as `

    `, `
  • ` or `

    `; a row's excerpt drops the marks. Titles and teasers stay one line through `strip_html`. List readers overwrite the description on every pull, so their roles take the new shape on the next pull; a detail reader reads a role's page once, so its roles read before 2026-09-30 keep the flat text. + **Readers scrape; `prep` decides.** A reader hands on the host's own words (`seniority`, `employment`, `remote`, and `country` only when the host states one per role) and does light cleanup only: text from HTML, dates, the places of a role as a list, pay figures. Mapping lived in 25 readers, each spelling its tables its own way, so changing what "Experienced" means touched a plugin. The `prep` stage (`app/prep/`) maps centrally after every pull and every enrich run. Seniority, employment and remote go through the host's table in `app/prep/hosts/.py` (whole words, or `MATCH` fragments by substring or token), then a shared word map. Country comes from the resolved places (several European countries give `EU`), and the host's country is only a fallback. Places and company aliases are resolved, and copies and duplicates are folded over the whole database. `upsert_role` stores the host words (`*_host` columns, migration 22) and marks the role `prep_due`. When a table changes, its fingerprint makes the next prep map every open role again from the stored words, with no pull. A word no table knows is listed in `unmapped` (`app prep unmapped`, the Sources page). A host's `remote: False` stays unknown rather than onsite, because SmartRecruiters, Ashby, Recruitee and Workable send it for hybrid roles too. **Unmapped places and words are fixed in code, from a list on Sources.** Settings › Sources › Unmapped lists the place texts of open roles no place resolves (`app prep unlinked`, with the countries and companies of their roles) and the host words no table maps. It has no editor: the owner tells Claude what a text is, and the `unmapped` skill in `.claude/skills` says which dictionary or host table takes it. Code rather than `add_place_alias` rows, because a fix then reaches a new database and is reviewed like any other change. The prep fingerprint covers `places.seed()`, so the next prep adds the new places and spellings to the database and maps every open role again. Copy the list puts both lists on the clipboard, or selects them on the page where there is no clipboard. diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 6134d17..17e5410 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -178,7 +178,7 @@ own id for the role, unique on the board), `title`, `url` (http or https, checked with `_shared.is_web_url`) and `places`; `location` (the host's one-line place, kept while storage reads it), `posted` (an aware datetime or None, see `_shared.parse_dt`), `department`, `description` (plain text, -see `desk.sources.strip_html`), `remote` (True, False, a word or None) and +see `_shared.html_text`), `remote` (True, False, a word or None) and `salary_text`; where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; where it states them, `country` (ISO 3166 alpha-2), `seniority` and diff --git a/plugins/readers/_shared.py b/plugins/readers/_shared.py index 2eb16ee..0828295 100644 --- a/plugins/readers/_shared.py +++ b/plugins/readers/_shared.py @@ -8,10 +8,12 @@ import it without either importing the other. The leading underscore keeps from __future__ import annotations +import html import re from collections.abc import Callable from datetime import UTC, datetime from email.utils import parsedate_to_datetime +from html.parser import HTMLParser from typing import Any, Protocol import httpx @@ -216,3 +218,123 @@ def add_place(found: list[Place], entry: Place) -> None: found.append(entry) + + +#: A description line that is a heading, and one that is a list item; what +#: the reader pane draws as `

    ` and `
  • `. The rest of a description is +#: plain text, one paragraph a line. +HEADING = "## " +BULLET = "\u2022 " + +#: Tags that end a line. A `
  • ` starts a bullet line, an `

    `..`

    ` a +#: heading line; the rest start a paragraph. +_BLOCKS = frozenset({ + "address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "fieldset", + "figcaption", "figure", "footer", "form", "header", "hr", "li", "main", "nav", "ol", "p", + "pre", "section", "table", "tbody", "thead", "tfoot", "tr", "ul", + "h1", "h2", "h3", "h4", "h5", "h6", +}) +_HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"}) +_BOLD = frozenset({"b", "strong"}) +_SKIPPED = frozenset({"script", "style", "head", "template", "noscript"}) +#: A paragraph set wholly in bold is a heading when it is at most this long; +#: longer, it is a paragraph somebody made loud. +_BOLD_HEADING = 80 +_TAG = re.compile(r"]*>") +_ESCAPED_TAG = re.compile(r"</?[A-Za-z]") +_ENTITY = re.compile(r"&(#\d+|#x[0-9A-Fa-f]+|[A-Za-z]+);") +#: A bullet typed as text: `- item`, `* item`, `\u00b7 item` and the like. +_TYPED_BULLET = re.compile("^[\u2022\u00b7\u25aa\u25cf\u25e6\u2023\u2043*\\-\u2013]\\s+") + + +class _Lines(HTMLParser): + """`html_text`'s parser: a line per block, each a kind ("p", "li" or + "h"), its text, and whether all of the text was bold.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.lines: list[tuple[str, str, bool]] = [] + self.kind = "p" + self.pieces: list[tuple[str, bool]] = [] + self.bold = 0 + self.skip = 0 + + def has_text(self) -> bool: + return any(text.strip() for text, _ in self.pieces) + + def flush(self) -> None: + if self.has_text(): + text = "".join(text for text, _ in self.pieces) + bold = all(b for text, b in self.pieces if text.strip()) + self.lines.append((self.kind, text, bold)) + self.pieces = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag in _SKIPPED: + self.skip += 1 + elif tag in _BOLD: + self.bold += 1 + elif tag in ("td", "th"): + self.pieces.append((" ", True)) + elif tag in _BLOCKS: + kind = "li" if tag == "li" else "h" if tag in _HEADINGS else "" + if self.has_text(): + self.flush() + self.kind = kind or "p" + elif kind: + self.kind = kind + # else `
  • text

  • `: the `

    ` keeps the bullet. + + def handle_endtag(self, tag: str) -> None: + if tag in _SKIPPED: + self.skip = max(0, self.skip - 1) + elif tag in _BOLD: + self.bold = max(0, self.bold - 1) + elif tag in _BLOCKS: + self.flush() + self.kind = "p" + + def handle_data(self, data: str) -> None: + if not self.skip: + self.pieces.append((data, self.bold > 0)) + + +def _line(kind: str, text: str, bold: bool) -> str: + text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\ufeff", "") + if _ENTITY.search(text): + # Escaped twice at the host (`&amp;`): once is left after parsing. + text = html.unescape(text) + text = " ".join(text.split()) + if typed := _TYPED_BULLET.match(text): + text, kind = text[typed.end():], "li" + if not text or text in ("\u2022", "\u00b7", "-", "*"): + return "" + if kind == "li": + return BULLET + text + if kind == "h" or (bold and len(text) <= _BOLD_HEADING): + return HEADING + text + return text + + +def html_text(markup: Any) -> str: + """A host's description HTML as plain text that keeps its shape: a line + per paragraph, `BULLET` before a list item and `HEADING` before a + heading (an `

    `..`

    `, or a short paragraph set wholly in bold). + Entities are decoded, and markup the host escaped once more + (`<p>`) is read as markup. Text with no tags keeps its own line + breaks, and a line typed as a bullet (`- item`) becomes one. + + This is a row's `description`: what the reader pane draws as headings, + lists and paragraphs, and what summarise and embed read. A one-line + field (a title, a teaser) stays `desk.sources.strip_html`.""" + markup = str(markup or "") + if not _TAG.search(markup): + if _ESCAPED_TAG.search(markup): + markup = html.unescape(markup) + else: + markup = "
    ".join(html.escape(line, quote=False) for line in markup.splitlines()) + parser = _Lines() + parser.feed(markup) + parser.close() + parser.flush() + return "\n".join(line for line in (_line(*entry) for entry in parser.lines) if line) diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py index e25bf15..9bfa138 100644 --- a/plugins/readers/ashby.py +++ b/plugins/readers/ashby.py @@ -12,10 +12,9 @@ import logging from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount +from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -142,7 +141,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": url, "posted": parse_dt(j.get("publishedAt")), "department": (j.get("department") or j.get("team") or "").strip(), - "description": strip_html(description), + "description": html_text(description), "remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too "salary_text": _salary_text(compensation), "salary_min": salary_min, diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py index dc9d031..838beaf 100644 --- a/plugins/readers/bamboohr.py +++ b/plugins/readers/bamboohr.py @@ -14,16 +14,14 @@ first) is listed like any other role. from __future__ import annotations -import html import json import logging from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, label_slug_ok, only, parse_dt, place +from ._shared import Get, html_text, label_slug_ok, only, parse_dt, place log = logging.getLogger(__name__) @@ -141,7 +139,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if not isinstance(opening, dict): raise ValueError("not a BambooHR role: no result.jobOpening") out: dict[str, Any] = { - "description": " ".join(html.unescape(strip_html(opening.get("description") or "")).split()), + "description": html_text(opening.get("description")), "posted": parse_dt(opening.get("datePosted")), } if place := _place(opening): diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py index a761ff0..a312a87 100644 --- a/plugins/readers/deel.py +++ b/plugins/readers/deel.py @@ -14,17 +14,15 @@ read of the role's page. from __future__ import annotations -import html import json import logging import re from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import num, only, parse_dt, pay_line +from ._shared import html_text, num, only, parse_dt, pay_line log = logging.getLogger(__name__) @@ -156,7 +154,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview", "posted": parse_dt(published.get("createdAt") or p.get("createdAt")), "department": ", ".join(dict.fromkeys(departments)), - "description": html.unescape(strip_html(description)), + "description": html_text(description), "remote": arrangement or None, "salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")), pay.get("currencyIsoCode"), "") diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index 0d776f7..3db4577 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -38,18 +38,16 @@ roles). from __future__ import annotations -import html import json import re from collections.abc import Callable from typing import Any from urllib.parse import quote -from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads, Row from ._countries import name_of -from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place +from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place READS = Reads( scope="board", listing="paged", page_size=10, text="detail", complete="stated", @@ -186,5 +184,4 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: data = answer.get("data") if isinstance(answer, dict) else None if not isinstance(data, dict) or "jobDescription" not in data: raise ValueError("not an Eightfold position: no data.jobDescription") - text = html.unescape(strip_html(data["jobDescription"] or "")) - return {"description": " ".join(text.split())} + return {"description": html_text(data["jobDescription"])} diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py index 1871833..696b5a0 100644 --- a/plugins/readers/gem.py +++ b/plugins/readers/gem.py @@ -17,17 +17,15 @@ The listing has no description and no date; both are on the role's page. from __future__ import annotations -import html import json import logging import re from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, add_place, only, parse_dt +from ._shared import Get, Page, add_place, html_text, only, parse_dt log = logging.getLogger(__name__) @@ -156,6 +154,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: {"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting") if not isinstance(role, dict): raise ValueError(f"Gem has no role {row['source_id']} on {slug}") - text = html.unescape(strip_html(role.get("descriptionHtml") or "")) - return {"description": " ".join(text.split()), + return {"description": html_text(role.get("descriptionHtml")), "posted": parse_dt(role.get("firstPublishedTsSec"))} diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py index 7d39b80..e851b04 100644 --- a/plugins/readers/greenhouse.py +++ b/plugins/readers/greenhouse.py @@ -6,16 +6,14 @@ The field map is ported from `openings.go` in from __future__ import annotations -import html import json import re from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place +from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place NAME = "greenhouse" LABEL = "Greenhouse" @@ -154,10 +152,6 @@ def parse(body: bytes, slug: str) -> list[Row]: if not title or not is_web_url(url): continue departments = j.get("departments") or [] - # Greenhouse ships the description HTML-entity-escaped inside the - # JSON string (`<p>`), so it needs unescaping before the tags - # themselves can be stripped. - content = html.unescape(j.get("content") or "") ranges = j.get("pay_input_ranges") or [] salary_min, salary_max, currency = _salary(ranges) places = _places(j) @@ -169,7 +163,9 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": url, "posted": parse_dt(j.get("first_published") or j.get("updated_at")), "department": (departments[0].get("name") if departments else "") or "", - "description": strip_html(content), + # Greenhouse ships the description HTML-entity-escaped inside the + # JSON string (`<p>`), which `html_text` reads as markup. + "description": html_text(j.get("content")), "remote": None, # Greenhouse's feed does not say "salary_text": _salary_text(ranges), "salary_min": salary_min, diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py index 2d65444..d87a4ab 100644 --- a/plugins/readers/hibob.py +++ b/plugins/readers/hibob.py @@ -14,16 +14,14 @@ filled, so the period's spelling is a guess. from __future__ import annotations -import html import json import logging from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Get, Page, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount log = logging.getLogger(__name__) @@ -98,7 +96,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: country = str(ad.get("country") or "").strip() location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \ else site or country - text = " ".join(strip_html(str(ad.get(key) or "")) for key in _SECTIONS) + text = "\n".join(part for key in _SECTIONS + if (part := html_text(ad.get(key)))) lo, hi = num(ad.get("payTransparencyMinSalary")), num(ad.get("payTransparencyMaxSalary")) currency = ad.get("payTransparencySalaryCurrency") period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "") @@ -114,7 +113,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: "url": f"https://{_host(slug)}/jobs/{ad_id}", "posted": parse_dt(ad.get("publishedAt")), "department": str(ad.get("department") or "").strip(), - "description": " ".join(html.unescape(text).split()), + "description": text, "remote": str(ad.get("workspaceTypeId") or "").strip() or None, "salary_text": pay_line(lo, hi, currency, period), "salary_min": salary_min, diff --git a/plugins/readers/hrmanager.py b/plugins/readers/hrmanager.py index e5c4836..d1a50e0 100644 --- a/plugins/readers/hrmanager.py +++ b/plugins/readers/hrmanager.py @@ -23,16 +23,14 @@ The feed names no place (the title or the text may), so every row has from __future__ import annotations -import html import re import xml.etree.ElementTree as ET from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, Page, is_web_url, only, parse_dt +from ._shared import Get, Page, html_text, is_web_url, only, parse_dt READS = Reads( scope="board", listing="paged", page_size=100, text="listing", complete="end", @@ -118,7 +116,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: "url": url, "posted": parse_dt(item.findtext("pubDate")), "department": "", - "description": html.unescape(strip_html(item.findtext("description") or "")), + "description": html_text(item.findtext("description")), "remote": None, "salary_text": "", }) diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py index 2e6d4f6..03f1a1e 100644 --- a/plugins/readers/jibe.py +++ b/plugins/readers/jibe.py @@ -28,17 +28,15 @@ any other site's, so the host is given by hand. from __future__ import annotations -import html import json import logging import re from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import Get, Page, add_place, only, parse_dt, place +from ._shared import Get, Page, add_place, html_text, only, parse_dt, place log = logging.getLogger(__name__) @@ -103,10 +101,6 @@ def parse(body: bytes, slug: str) -> list[Row]: return rows(json.loads(body), slug) -def _plain(markup: Any) -> str: - return " ".join(html.unescape(strip_html(str(markup or ""))).split()) - - def _places(j: dict[str, Any]) -> list[Place]: """`full_location` names every place of the role, "; " apart, each as "City, Country"; the site's `city` and `country` stand in when it is @@ -136,8 +130,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: if not title or not role: continue categories = j.get("categories") or [] - text = "\n\n".join(p for p in (_plain(j.get("description")), - _plain(j.get("qualifications"))) if p) + text = "\n".join(p for p in (html_text(j.get("description")), + html_text(j.get("qualifications"))) if p) places = _places(j) row: Row = { "source_id": role, diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py index cf063bb..477461c 100644 --- a/plugins/readers/jobylon.py +++ b/plugins/readers/jobylon.py @@ -26,7 +26,7 @@ from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, parse_dt, place +from ._shared import Get, html_text, parse_dt, place log = logging.getLogger(__name__) @@ -227,7 +227,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if name and name not in places: places.append(name) found: dict[str, Any] = { - "description": html.unescape(strip_html(data.get("description") or "")), + "description": html_text(data.get("description")), "posted": parse_dt(data.get("datePosted")), } if places: diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index 90256f2..7ee003b 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -20,10 +20,9 @@ import logging from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount +from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -137,7 +136,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": url, "posted": parse_dt(j.get("createdAt")), "department": (categories.get("department") or categories.get("team") or "").strip(), - "description": strip_html(description), + "description": html_text(description), "remote": (j.get("workplaceType") or "").strip() or None, "salary_text": _salary_text(salary_range), "salary_min": salary_min, diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index c61b985..4762c82 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -39,7 +39,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES, code_of -from ._shared import Get, Page, Row, add_place, only, parse_dt +from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt NAME = "oracle" LABEL = "Oracle Recruiting Cloud" @@ -231,9 +231,13 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if not items: return {} role = items[0] - text = " ".join(part for key in ("ExternalDescriptionStr", "ExternalResponsibilitiesStr", - "ExternalQualificationsStr") - if (part := _plain(role.get(key)))) + parts = [] + for key, heading in (("ExternalDescriptionStr", ""), + ("ExternalResponsibilitiesStr", "Responsibilities"), + ("ExternalQualificationsStr", "Qualifications")): + if part := html_text(_text(role.get(key))): + parts += [HEADING + heading, part] if heading else [part] + text = "\n".join(parts) out: dict[str, Any] = {} if text: out["description"] = text diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py index bb34b4d..070e4fd 100644 --- a/plugins/readers/personio.py +++ b/plugins/readers/personio.py @@ -10,10 +10,9 @@ from __future__ import annotations import xml.etree.ElementTree as ET from collections.abc import Callable -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount +from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount NAME = "personio" LABEL = "Personio" @@ -128,7 +127,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": f"https://{slug}.jobs.personio.de/job/{job_id}", "posted": parse_dt(position.findtext("createdAt")), "department": (position.findtext("department") or "").strip(), - "description": strip_html(description), + "description": html_text(description), "seniority": (position.findtext("seniority") or "").strip() or None, "employment": _employment(position), "salary_text": salary_text, diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index 944255a..e49a101 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -35,7 +35,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import name_of -from ._shared import Get, Page, Row, add_place, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt NAME = "phenom" LABEL = "Phenom" @@ -243,8 +243,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace") posting, job = _job_posting(body), _page_job(body) out: dict[str, Any] = {} - if description := (_plain(html.unescape(_text(posting.get("description")))) - or _plain(job.get("description"))): + if description := (html_text(html.unescape(_text(posting.get("description")))) + or html_text(_text(job.get("description")))): out["description"] = description else: raise ValueError("no description on the role's page") diff --git a/plugins/readers/pinpoint.py b/plugins/readers/pinpoint.py index f59fef9..cd4554c 100644 --- a/plugins/readers/pinpoint.py +++ b/plugins/readers/pinpoint.py @@ -16,15 +16,13 @@ so `posted` stays None rather than costing a request a role for it. from __future__ import annotations -import html import json from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Reads -from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount +from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount NAME = "pinpoint" LABEL = "Pinpoint" @@ -70,22 +68,17 @@ def feed_url(slug: str, ask: Ask = Ask()) -> str: return f"https://{host(slug)}/postings.json" -def _plain(markup: Any) -> str: - """HTML as one line of plain text: the tags out, the entities read (Pinpoint - writes `&` and ` `), the whitespace, non-breaking included, collapsed.""" - return " ".join(html.unescape(strip_html(str(markup or ""))).split()) - - def _text(j: dict[str, Any]) -> str: """The description and the sections after it, each section under its own - heading, as plain text with a blank line between them.""" - parts = [_plain(j.get("description"))] + heading, as `html_text` lines.""" + parts = [html_text(j.get("description"))] for field in _SECTIONS: - body = _plain(j.get(field)) + body = html_text(j.get(field)) if body: - heading = str(j.get(f"{field}_header") or "").strip() - parts.append(f"{heading}: {body}" if heading else body) - return "\n\n".join(p for p in parts if p) + if heading := " ".join(str(j.get(f"{field}_header") or "").split()): + parts.append(HEADING + heading) + parts.append(body) + return "\n".join(p for p in parts if p) def _place(location: dict[str, Any]) -> str: diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py index 19e3183..3382dd1 100644 --- a/plugins/readers/platsbanken.py +++ b/plugins/readers/platsbanken.py @@ -22,7 +22,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt NAME = "platsbanken" LABEL = "Platsbanken" @@ -124,6 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: "country": "SE", "remote": model or None, "posted": parse_dt(hit.get("publication_date")), - "description": _text((hit.get("description") or {}).get("text") or ""), + "description": html_text((hit.get("description") or {}).get("text")), }) return out diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py index cce230b..e4c396c 100644 --- a/plugins/readers/recruitee.py +++ b/plugins/readers/recruitee.py @@ -11,10 +11,9 @@ import json from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount NAME = "recruitee" LABEL = "Recruitee" @@ -121,7 +120,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "url": url, "posted": parse_dt(j.get("published_at") or j.get("created_at")), "department": (j.get("department") or "").strip(), - "description": strip_html(j.get("description") or ""), + "description": html_text(j.get("description")), "remote": j.get("remote"), "salary_text": _salary_text(salary), "salary_min": salary_min, diff --git a/plugins/readers/rippling.py b/plugins/readers/rippling.py index 1e98e4d..2104fb3 100644 --- a/plugins/readers/rippling.py +++ b/plugins/readers/rippling.py @@ -10,16 +10,14 @@ Board not found"). from __future__ import annotations -import html import json import re from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Row, add_place, parse_dt, yearly_amount +from ._shared import Get, Row, add_place, html_text, parse_dt, yearly_amount NAME = "rippling" LABEL = "Rippling" @@ -122,7 +120,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: data = json.loads(get(f"{_board(slug)}/jobs/{row['source_id']}")) text = data.get("description") or {} found: dict[str, Any] = { - "description": html.unescape(strip_html(text.get("role") or text.get("company") or "")), + "description": html_text(text.get("role") or text.get("company")), "posted": parse_dt(data.get("createdOn")), } if names := [p for p in (str(p).strip() for p in data.get("workLocations") or []) if p]: diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index f8bb5a9..e433f0b 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -24,17 +24,15 @@ company blurb is the same on every role of the board and is left out. from __future__ import annotations -import html import json import re from collections.abc import Callable from typing import Any from urllib.parse import quote -from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok +from ._shared import Get, HEADING, Page, Row, html_text, only, parse_dt, path_slug_ok NAME = "smartrecruiters" LABEL = "SmartRecruiters" @@ -162,8 +160,9 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: raise ValueError("not a SmartRecruiters posting: no jobAd.sections") parts = [] for key in _SECTIONS: - text = " ".join(html.unescape(strip_html(str((sections.get(key) or {}).get("text") or ""))) - .split()) - if text: + section = sections.get(key) or {} + if text := html_text(section.get("text")): + if title := " ".join(str(section.get("title") or "").split()): + parts.append(HEADING + title) parts.append(text) - return {"description": " ".join(parts)} + return {"description": "\n".join(parts)} diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 71c2f9a..03dfee2 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -63,7 +63,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES -from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt NAME = "successfactors" LABEL = "SuccessFactors" @@ -375,7 +375,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: # An employer's own layout can put a style block in the description. block = re.sub(r"<(style|script)\b.*?", " ", _element(text, match.start()), flags=re.S | re.I) - if description := _text(block): + if description := html_text(block): out["description"] = description if re.search(r'itemtype="https?://schema\.org/JobPosting"', text): meta = {name: html.unescape(value) for name, value in _META.findall(text)} diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py index 328fc6c..cb4d7fc 100644 --- a/plugins/readers/teamtailor.py +++ b/plugins/readers/teamtailor.py @@ -8,16 +8,14 @@ Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor. from __future__ import annotations -import html import re import xml.etree.ElementTree as ET from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt NAME = "teamtailor" LABEL = "Teamtailor" @@ -136,7 +134,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: "posted": parse_dt(item.findtext("pubDate")), "department": ((item.findtext(f"{_TT}department") or "").strip() or (item.findtext(f"{_TT}role") or "").strip()), - "description": html.unescape(strip_html(item.findtext("description") or "")), + "description": html_text(item.findtext("description")), "remote": status or None, "salary_text": "", # Teamtailor's feed states no pay at all }) diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py index 36aa055..28b3d6e 100644 --- a/plugins/readers/workable.py +++ b/plugins/readers/workable.py @@ -15,15 +15,13 @@ reads and stops on one. from __future__ import annotations -import html import json from collections.abc import Callable from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok +from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt, path_slug_ok NAME = "workable" LABEL = "Workable" @@ -146,7 +144,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: raise ValueError("not a Workable role: no description") parts = [] for key in _PARTS: - text = " ".join(html.unescape(strip_html(str(data.get(key) or ""))).split()) - if text: + if text := html_text(data.get(key)): + if key != "description": + parts.append(HEADING + key.capitalize()) parts.append(text) - return {"description": " ".join(parts)} + return {"description": "\n".join(parts)} diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index 02a253f..0cbebcf 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -31,18 +31,16 @@ A fetch term goes as `searchText` in the body, one a request from __future__ import annotations -import html import json import re from collections.abc import Callable from datetime import UTC, datetime, timedelta from typing import Any -from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import code_of -from ._shared import Get, Page, Row, add_place, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt NAME = "workday" LABEL = "Workday" @@ -316,8 +314,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: if not isinstance(posting, dict): raise ValueError("not a Workday role: no jobPostingInfo") out: dict[str, Any] = {} - if description := " ".join(html.unescape(strip_html(posting.get("jobDescription") or "")) - .split()): + if description := html_text(posting.get("jobDescription")): out["description"] = description remote = _remote(posting.get("remoteType")) if places := _places(posting, remote): diff --git a/tests/test_ats.py b/tests/test_ats.py index 74d5b1a..e82808c 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -276,7 +276,7 @@ def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None: first = rows[0] assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer" assert first["department"] == "Engineering" - assert first["description"] == "About the role Build & run care." + assert first["description"] == "## About the role\nBuild & run care." assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC) assert first["salary_text"] == "" diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index 09f17d4..e2c27c6 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -113,7 +113,7 @@ def test_detail_of_the_standing_invitation() -> None: url = "https://configit.bamboohr.com/careers/26/detail" found = bamboohr.detail(_get_of({url: _read("bamboohr-detail-26.json")}, []), "configit", row) assert found["posted"] == datetime(2025, 10, 10, tzinfo=UTC) - assert found["description"].startswith("Hi, If there is currently no vacant position") + assert found["description"].startswith("Hi,\nIf there is currently no vacant position") assert "UK (remote)" in found["description"] diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py index 7a313e4..38c230b 100644 --- a/tests/test_ats_deel.py +++ b/tests/test_ats_deel.py @@ -33,7 +33,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None: "21584046-56d5-44c4-8b18-7e15dc659827/overview") assert first["location"] == "Stockholm" and first["department"] == "Engineering" assert first["remote"] == "ON_SITE" - assert first["description"].startswith("Klarna, briefly At Klarna") + assert first["description"].startswith("## Klarna, briefly\nAt Klarna") assert "<" not in first["description"] and "$" not in first["description"][:3] assert all(r["description"] for r in rows) # The day the posting was published, not the day it was drafted (12:27). diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index 266098b..4d55f56 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -151,7 +151,7 @@ def test_detail_reads_the_text_of_the_ad() -> None: "?position_id=563121777283313&domain=ericsson.com&hl=en"] assert set(found) <= set(ats.DETAIL_FIELDS) text = found["description"] - assert text.startswith("Join our Team") + assert text.startswith("## Join our Team\n") assert "<" not in text and "&" not in text and "\r" not in text and " " not in text diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index c5f76bf..96579d0 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -129,7 +129,7 @@ def test_detail_reads_the_text_and_the_date() -> None: assert asked[0]["query"] == gem.DETAIL_QUERY assert set(found) <= set(ats.DETAIL_FIELDS) assert found["posted"] == datetime.fromtimestamp(1787574069, tz=UTC) - assert found["description"].startswith("Only the best engineers are being built here.") + assert found["description"].startswith("## Only the best engineers are being built here.\n") assert "<" not in found["description"] and "&" not in found["description"] diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py index 1a1aa1e..3198948 100644 --- a/tests/test_ats_hrmanager.py +++ b/tests/test_ats_hrmanager.py @@ -36,7 +36,7 @@ def test_rows_read_the_advertisement_and_the_publication_date() -> None: assert first["title"] == "Commissioning Engineer- T&D" # `&` in the XML assert first["url"] == ("https://candidate.hr-manager.net/ApplicationInit.aspx" "?cid=2523&ProjectId=143857&DepartmentId=18958&MediaId=5") - assert first["description"].startswith("About the Position We have an exciting opportunity") + assert first["description"].startswith("## About the Position\nWe have an exciting opportunity") assert "<" not in first["description"] and "&" not in first["description"] assert "T&D Division" in first["description"] assert first["posted"] is not None and first["posted"].isoformat() == \ diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 05568c4..6de4180 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -64,7 +64,7 @@ def test_a_page_becomes_rows() -> None: def test_qualifications_follow_the_description() -> None: text = jibe.rows(_saved(), SITE)[-1]["description"] - assert "\n\n" in text and text.split("\n\n")[-1].startswith("Java") + assert any(line.startswith("Java") for line in text.split("\n")[1:]) def test_the_location_falls_back_to_city_and_country() -> None: diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py index aba4256..f12d433 100644 --- a/tests/test_ats_oracle.py +++ b/tests/test_ats_oracle.py @@ -195,7 +195,7 @@ def test_detail_appends_the_responsibilities_and_qualifications_where_filled() - data["items"][0]["ExternalResponsibilitiesStr"] = "
    • Run the desk
    " data["items"][0]["ExternalQualificationsStr"] = "

    Five years & more

    " got = oracle.detail(_detail_get(json.dumps(data).encode(), []), SLUG, {"source_id": "24594"}) - assert got["description"].endswith(" Run the desk Five years & more") + assert got["description"].endswith("\n## Responsibilities\n\u2022 Run the desk\n## Qualifications\nFive years & more") def test_detail_of_a_role_no_longer_open_has_nothing_to_read() -> None: diff --git a/tests/test_ats_phenom.py b/tests/test_ats_phenom.py index a6aee31..97ef8e8 100644 --- a/tests/test_ats_phenom.py +++ b/tests/test_ats_phenom.py @@ -205,7 +205,7 @@ def test_detail_reads_the_description_from_the_json_ld() -> None: row = _rows()[0] got = phenom.detail(_get_of("phenom-detail.html", asked), SLUG, row) assert asked == [(row["url"], {"Accept": "text/html"})] - assert got["description"].startswith("Craft: Data, Analytics & Strategy") + assert got["description"].startswith("## Craft:\nData, Analytics & Strategy") assert "<" not in got["description"] and "<" not in got["description"] assert "&" not in got["description"] assert got["posted"].isoformat() == "2026-09-28T00:00:00+00:00" @@ -215,7 +215,7 @@ def test_detail_reads_the_description_from_the_json_ld() -> None: def test_detail_finds_the_job_posting_among_other_blocks_and_takes_its_date() -> None: got = phenom.detail(_get_of("phenom-detail-2.html", []), "www.careers.philips.com/global/en", {"url": "https://www.careers.philips.com/global/en/job/1/x"}) - assert got["description"].startswith("Job Title Zone Business Leader Imaging Nordics") + assert got["description"].startswith("## Job Title\nZone Business Leader Imaging Nordics") assert got["posted"].isoformat() == "2026-09-08T00:00:00+00:00" @@ -224,7 +224,7 @@ def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() -> `WebPage` one, and the role in the state the page assigns to `phApp.ddo`.""" row = _rows()[0] got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row) - assert got["description"].startswith("Craft: Games Design & Production Job Description:") + assert got["description"].startswith("## Craft:\nGames Design & Production\n## Job Description:") assert "<" not in got["description"] and "&" not in got["description"] assert got["remote"] == "On-site" assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS) diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py index b3741de..d1756ae 100644 --- a/tests/test_ats_pinpoint.py +++ b/tests/test_ats_pinpoint.py @@ -39,13 +39,13 @@ def test_a_listing_becomes_rows() -> None: first["currency"]) == ("", None, None, "") -def test_the_description_joins_the_sections_as_plain_text() -> None: +def test_the_description_puts_each_section_under_its_heading() -> None: text = pinpoint.parse(_body(), "xeneta")[0]["description"] - assert text.startswith("Data Detective Wanted to join our Data & Analytics Team") + assert text.startswith("## Data Detective Wanted to join our Data & Analytics Team\n") assert "<" not in text and "&" not in text and "\xa0" not in text - assert "\n\nKey Responsibilities: Join our Liner Data & Analytics Team" in text - assert "\n\nSkills, Knowledge & Expertise: In this role" in text - assert "\n\nJob Benefits: We will help you grow your career:" in text + assert "\n## Key Responsibilities\nJoin our Liner Data & Analytics Team" in text + assert "\n## Skills, Knowledge & Expertise\nIn this role" in text + assert "\n## Job Benefits\n\u2022 We will help you grow your career:" in text def test_a_region_is_kept_when_it_says_more_than_the_name() -> None: diff --git a/tests/test_ats_rippling.py b/tests/test_ats_rippling.py index ed5a89c..eb0c932 100644 --- a/tests/test_ats_rippling.py +++ b/tests/test_ats_rippling.py @@ -85,7 +85,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked), "rippling", row) assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"] - assert found["description"].startswith("About The Role") + assert found["description"].startswith("## About The Role\n") assert "<" not in found["description"] assert found["location"] == "San Francisco, CA; Seattle, WA; New York, NY" assert [p["text"] for p in found["places"]] == ["San Francisco, CA", "Seattle, WA", "New York, NY"] @@ -98,7 +98,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non def test_detail_without_pay_leaves_the_pay_columns_out() -> None: found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()), "prisma-careers", {"source_id": "x"}) - assert found["description"].startswith("What is it like to work at Prisma?") + assert found["description"].startswith("## What is it like to work at Prisma?\n") assert found["location"] == "Remote (Germany)" assert found["places"] == [{"text": "Remote (Germany)", "remote": True}] assert not {"salary_min", "salary_max", "currency"} & set(found) diff --git a/tests/test_ats_smartrecruiters.py b/tests/test_ats_smartrecruiters.py index 8a829ba..f53c778 100644 --- a/tests/test_ats_smartrecruiters.py +++ b/tests/test_ats_smartrecruiters.py @@ -103,7 +103,7 @@ def test_detail_leaves_out_a_section_the_ad_lacks() -> None: body = json.dumps({"jobAd": {"sections": { "jobDescription": {"text": "

    Build & run

    • APIs
    "}}}}).encode() assert smartrecruiters.detail(lambda url, **_: body, "Acme", {"source_id": "7"}) == \ - {"description": "Build & run APIs"} + {"description": "Build & run\n\u2022 APIs"} @pytest.mark.parametrize("body", [b"{}", b'{"jobAd": {}}', b'{"jobAd": null}', b"[]"]) diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py index 9e0fb33..c3b4e4b 100644 --- a/tests/test_ats_successfactors.py +++ b/tests/test_ats_successfactors.py @@ -317,7 +317,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/" def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None: got = _detail("successfactors-job.html") - assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you") + assert got["description"].startswith("## Senior Management | Full-Time | Commercial\nAre you") assert "<" not in got["description"] # `datePosted` is a Java date, "Tue Sep 29 02:00:00 UTC 2026". assert got["posted"] == datetime(2026, 9, 29, 2, tzinfo=UTC) @@ -347,14 +347,14 @@ def test_detail_reads_the_place_from_a_street_address() -> None: got = _detail("successfactors-job-street.html") assert got["location"] == "Bern, Switzerland" assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC) - assert got["description"].startswith("Senior Solution Engineer Defence") + assert got["description"].startswith("## Senior Solution Engineer Defence") def test_detail_of_a_page_without_microdata_gives_the_description_only() -> None: """Novo Nordisk: the listing's date and location stand.""" got = _detail("successfactors-job-plain.html") assert set(got) == {"description"} - assert got["description"].startswith("Process Supporter, Koriyama Factory") + assert got["description"].startswith("## Process Supporter, Koriyama Factory\n") assert "Your new role" in got["description"] @@ -364,7 +364,7 @@ def test_detail_leaves_a_style_block_out_of_the_description() -> None: 'Nested

    Footer

    ') got = successfactors.detail(lambda url, **kw: page.encode(), "careers.acme.com", {"url": "https://careers.acme.com/job/x/1/"}) - assert got == {"description": "Build things. Nested"} + assert got == {"description": "Build things.\nNested"} def test_detail_of_a_page_that_is_not_a_role_is_empty() -> None: @@ -385,7 +385,7 @@ def test_detail_is_read_through_the_shared_code_for_the_roles_not_read( assert done == {f"successfactors:careers.vestas.com:{r['source_id']}" for r in rows} assert not failed assert out[0]["location"] == "Seoul, South Korea" - assert out[0]["description"].startswith("Senior Management") + assert out[0]["description"].startswith("## Senior Management") assert out[0]["title"] == rows[0]["title"] diff --git a/tests/test_ats_workable.py b/tests/test_ats_workable.py index 6a71de2..f3cc3bd 100644 --- a/tests/test_ats_workable.py +++ b/tests/test_ats_workable.py @@ -15,8 +15,8 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from desk.sources import strip_html # noqa: E402 from plugins.readers import Ask, _runner, workable # noqa: E402 +from plugins.readers._shared import html_text # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -86,9 +86,8 @@ def test_detail_reads_description_requirements_and_benefits() -> None: assert set(found) == {"description"} and set(found) <= set(_runner.DETAIL_FIELDS) text, data = found["description"], json.loads(body) for key in ("description", "requirements", "benefits"): - assert strip_html(data[key])[:40] in text - assert text.index(strip_html(data["description"])[:40]) < \ - text.index(strip_html(data["benefits"])[:40]) + assert html_text(data[key]) in text + assert text.index(html_text(data["description"])) < text.index(html_text(data["benefits"])) assert "<" not in text and " " not in text and len(text) > 1500 @@ -96,7 +95,7 @@ def test_detail_joins_what_is_there_and_unescapes() -> None: body = json.dumps({"description": "

    Build & run

    ", "requirements": None, "benefits": "
    • Pension
    "}).encode() assert workable.detail(lambda url, **_: body, "acme", {"shortcode": "AAA111"}) == \ - {"description": "Build & run Pension"} + {"description": "Build & run\n## Benefits\n\u2022 Pension"} @pytest.mark.parametrize("body", [b"{}", b"[]", b'{"title": "x"}']) diff --git a/tests/test_readers_text.py b/tests/test_readers_text.py new file mode 100644 index 0000000..5b7b431 --- /dev/null +++ b/tests/test_readers_text.py @@ -0,0 +1,47 @@ +"""`html_text`: a host's description HTML as lines that keep its headings, +bullets and paragraphs, and `excerpt` leaving those marks out of a row.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.models import excerpt # noqa: E402 +from plugins.readers._shared import html_text # noqa: E402 + + +def test_blocks_become_lines_lists_bullets_and_headings_marks() -> None: + markup = ("

    About

    We are great.

    " + "

    What you will do:

    " + "
    • Build & ship

    • Lead
      • nested
    " + "

    Line one
    Line two

    ") + assert html_text(markup) == ("## About\nWe are great.\n## What you will do:\n" + "• Build & ship\n• Lead\n• nested\n" + "Line one\nLine two") + + +def test_a_long_bold_paragraph_is_not_a_heading() -> None: + loud = "

    " + "Apply now " * 10 + "

    " + assert not html_text(loud).startswith("## ") + + +def test_markup_escaped_once_more_is_read_as_markup() -> None: + assert html_text("<p>Hello &amp; bye</p><ul><li>a</li></ul>") \ + == "Hello & bye\n• a" + + +def test_plain_text_keeps_its_lines_and_typed_bullets_become_bullets() -> None: + assert html_text("plain text\n\n* one\n- two\nR&D < 5") == \ + "plain text\n• one\n• two\nR&D < 5" + + +def test_nothing_gives_nothing() -> None: + assert html_text(None) == html_text("") == html_text("

    ") == "" + + +def test_the_row_excerpt_leaves_the_marks_out() -> None: + assert excerpt("## About the role\n• Build & run care.") == "About the role Build & run care."