diff --git a/app/models.py b/app/models.py index f9fb696..c94de92 100644 --- a/app/models.py +++ b/app/models.py @@ -1020,14 +1020,18 @@ class ScoredRole(Role): #: carries more of a role's text than a row draws. EXCERPT_LIMIT = 200 +#: The marks `html_text` starts a heading and a list item with. +_LINE_MARK = re.compile("^(## |\u2022 )", re.M) + def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str: """`text` (a role's summary, or the opening of its description) with its whitespace collapsed, cut to `limit` characters at a word boundary with an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a `SUBSTR` of the source column rather than the whole of it, so the feed's - list never carries what only the reader needs.""" - collapsed = " ".join((text or "").split()) + list never carries what only the reader needs. A description's heading + and bullet marks (`plugins.readers._shared.html_text`) are left out.""" + collapsed = " ".join(_LINE_MARK.sub("", text or "").split()) if len(collapsed) <= limit: return collapsed cut = collapsed[:limit] diff --git a/app/static/app.css b/app/static/app.css index 9c83a62..e5e5e10 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -929,6 +929,9 @@ body[data-reader="open"] .company-strip { display: none; } margin-top: 1.3rem; font-size: 1.06rem; line-height: 1.7; } .prose p { margin: 0 0 1.05rem; } +/* A description's headings and bullets (roles.js `paragraphs`), newsdesk's rules. */ +.prose h2, .prose h3, .prose h4 { line-height: 1.3; margin: 1.4rem 0 .7rem; } +.prose ul, .prose ol { padding-left: 1.5rem; } .reader-out { margin-top: 1.5rem; padding-top: 1rem; border-top: 1px solid var(--line); } .reader-note a { color: var(--accent); } .reader-copy { margin-top: .8rem; } diff --git a/app/static/roles.js b/app/static/roles.js index f2da6e7..aa77dc8 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -861,13 +861,17 @@ return role.excerpt || ""; } + /* The heading and bullet marks of a description line, see `paragraphs`. */ + const LINE_MARK = /^(## |\u2022 )/gm; + /* The same cut, run here instead: for the one row `merge` refreshes after a fetch or a summarise, off the full role that call answers with, which is the only time this page ever sees a role's `summary` or `description` directly. */ function excerptOf(role) { /* The server's order (`_EXCERPT_SOURCE`): summary, description, teaser. */ - const text = (role.summary || role.description || role.teaser || "").replace(/\s+/g, " ").trim(); + const text = (role.summary || role.description || role.teaser || "") + .replace(LINE_MARK, "").replace(/\s+/g, " ").trim(); if (text.length <= 200) return text; const cut = text.slice(0, 200); const space = cut.lastIndexOf(" "); @@ -1291,11 +1295,22 @@ try { return new URL(url).host.replace(/^www\./, ""); } catch (e) { return ""; } } - /* Plain text in paragraphs: a fetched page keeps its line breaks, and a - board's description is one long line, which stays one paragraph. */ + /* A description is a line per paragraph (`plugins.readers._shared.html_text`): + "## " starts a heading and "• " a list item, which is how a board's HTML + keeps its sections and bullets (`LINE_MARK`). A fetched page keeps its + line breaks, and an older description is one long line, which stays one + paragraph. */ function paragraphs(text) { - return text.split(/\n+/).map((p) => p.trim()).filter(Boolean) - .map((p) => `
${esc(p)}
`).join(""); + let html = ""; + let list = false; + for (const line of text.split(/\n+/).map((p) => p.trim()).filter(Boolean)) { + const item = line.startsWith("\u2022 "); + if (item !== list) { html += item ? "${esc(line)}
`; + } + return list ? `${html}` : html; } function renderReader({ loading = false, error = "" } = {}) { diff --git a/docs/decisions.md b/docs/decisions.md index 03006f2..fa98cc0 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -130,6 +130,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A list teaser is kept.** A short description in a listing (Phenom, Oracle, Jobylon, Jobindex, Adzuna) is `roles.teaser`, shown and embedded until the full text arrives, when the vector is dropped and the text replaces it. +**A description keeps its shape as marked lines.** `_shared.html_text` turns a host's HTML into a line per paragraph, `• ` before a list item and `## ` before a heading (an ``; a row's excerpt drops the marks. Titles and teasers stay one line through `strip_html`. List readers overwrite the description on every pull, so their roles take the new shape on the next pull; a detail reader reads a role's page once, so its roles read before 2026-09-30 keep the flat text.
+
**Readers scrape; `prep` decides.** A reader hands on the host's own words (`seniority`, `employment`, `remote`, and `country` only when the host states one per role) and does light cleanup only: text from HTML, dates, the places of a role as a list, pay figures. Mapping lived in 25 readers, each spelling its tables its own way, so changing what "Experienced" means touched a plugin. The `prep` stage (`app/prep/`) maps centrally after every pull and every enrich run. Seniority, employment and remote go through the host's table in `app/prep/hosts/ text ` keeps the bullet.
+
+ def handle_endtag(self, tag: str) -> None:
+ if tag in _SKIPPED:
+ self.skip = max(0, self.skip - 1)
+ elif tag in _BOLD:
+ self.bold = max(0, self.bold - 1)
+ elif tag in _BLOCKS:
+ self.flush()
+ self.kind = "p"
+
+ def handle_data(self, data: str) -> None:
+ if not self.skip:
+ self.pieces.append((data, self.bold > 0))
+
+
+def _line(kind: str, text: str, bold: bool) -> str:
+ text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\ufeff", "")
+ if _ENTITY.search(text):
+ # Escaped twice at the host (`&`): once is left after parsing.
+ text = html.unescape(text)
+ text = " ".join(text.split())
+ if typed := _TYPED_BULLET.match(text):
+ text, kind = text[typed.end():], "li"
+ if not text or text in ("\u2022", "\u00b7", "-", "*"):
+ return ""
+ if kind == "li":
+ return BULLET + text
+ if kind == "h" or (bold and len(text) <= _BOLD_HEADING):
+ return HEADING + text
+ return text
+
+
+def html_text(markup: Any) -> str:
+ """A host's description HTML as plain text that keeps its shape: a line
+ per paragraph, `BULLET` before a list item and `HEADING` before a
+ heading (an ` Five years & more Build & run Footer Build & run We are great. What you will do: Build & ship Line one " + "Apply now " * 10 + " ` and `
`..`
` a
+#: heading line; the rest start a paragraph.
+_BLOCKS = frozenset({
+ "address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "fieldset",
+ "figcaption", "figure", "footer", "form", "header", "hr", "li", "main", "nav", "ol", "p",
+ "pre", "section", "table", "tbody", "thead", "tfoot", "tr", "ul",
+ "h1", "h2", "h3", "h4", "h5", "h6",
+})
+_HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
+_BOLD = frozenset({"b", "strong"})
+_SKIPPED = frozenset({"script", "style", "head", "template", "noscript"})
+#: A paragraph set wholly in bold is a heading when it is at most this long;
+#: longer, it is a paragraph somebody made loud.
+_BOLD_HEADING = 80
+_TAG = re.compile(r"?[A-Za-z][^>]*>")
+_ESCAPED_TAG = re.compile(r"</?[A-Za-z]")
+_ENTITY = re.compile(r"&(#\d+|#x[0-9A-Fa-f]+|[A-Za-z]+);")
+#: A bullet typed as text: `- item`, `* item`, `\u00b7 item` and the like.
+_TYPED_BULLET = re.compile("^[\u2022\u00b7\u25aa\u25cf\u25e6\u2023\u2043*\\-\u2013]\\s+")
+
+
+class _Lines(HTMLParser):
+ """`html_text`'s parser: a line per block, each a kind ("p", "li" or
+ "h"), its text, and whether all of the text was bold."""
+
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self.lines: list[tuple[str, str, bool]] = []
+ self.kind = "p"
+ self.pieces: list[tuple[str, bool]] = []
+ self.bold = 0
+ self.skip = 0
+
+ def has_text(self) -> bool:
+ return any(text.strip() for text, _ in self.pieces)
+
+ def flush(self) -> None:
+ if self.has_text():
+ text = "".join(text for text, _ in self.pieces)
+ bold = all(b for text, b in self.pieces if text.strip())
+ self.lines.append((self.kind, text, bold))
+ self.pieces = []
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if tag in _SKIPPED:
+ self.skip += 1
+ elif tag in _BOLD:
+ self.bold += 1
+ elif tag in ("td", "th"):
+ self.pieces.append((" ", True))
+ elif tag in _BLOCKS:
+ kind = "li" if tag == "li" else "h" if tag in _HEADINGS else ""
+ if self.has_text():
+ self.flush()
+ self.kind = kind or "p"
+ elif kind:
+ self.kind = kind
+ # else `
`..`
`, or a short paragraph set wholly in bold).
+ Entities are decoded, and markup the host escaped once more
+ (`<p>`) is read as markup. Text with no tags keeps its own line
+ breaks, and a line typed as a bullet (`- item`) becomes one.
+
+ This is a row's `description`: what the reader pane draws as headings,
+ lists and paragraphs, and what summarise and embed read. A one-line
+ field (a title, a teaser) stays `desk.sources.strip_html`."""
+ markup = str(markup or "")
+ if not _TAG.search(markup):
+ if _ESCAPED_TAG.search(markup):
+ markup = html.unescape(markup)
+ else:
+ markup = "
".join(html.escape(line, quote=False) for line in markup.splitlines())
+ parser = _Lines()
+ parser.feed(markup)
+ parser.close()
+ parser.flush()
+ return "\n".join(line for line in (_line(*entry) for entry in parser.lines) if line)
diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py
index e25bf15..9bfa138 100644
--- a/plugins/readers/ashby.py
+++ b/plugins/readers/ashby.py
@@ -12,10 +12,9 @@ import logging
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
+from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -142,7 +141,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": url,
"posted": parse_dt(j.get("publishedAt")),
"department": (j.get("department") or j.get("team") or "").strip(),
- "description": strip_html(description),
+ "description": html_text(description),
"remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too
"salary_text": _salary_text(compensation),
"salary_min": salary_min,
diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py
index dc9d031..838beaf 100644
--- a/plugins/readers/bamboohr.py
+++ b/plugins/readers/bamboohr.py
@@ -14,16 +14,14 @@ first) is listed like any other role.
from __future__ import annotations
-import html
import json
import logging
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import Get, label_slug_ok, only, parse_dt, place
+from ._shared import Get, html_text, label_slug_ok, only, parse_dt, place
log = logging.getLogger(__name__)
@@ -141,7 +139,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if not isinstance(opening, dict):
raise ValueError("not a BambooHR role: no result.jobOpening")
out: dict[str, Any] = {
- "description": " ".join(html.unescape(strip_html(opening.get("description") or "")).split()),
+ "description": html_text(opening.get("description")),
"posted": parse_dt(opening.get("datePosted")),
}
if place := _place(opening):
diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py
index a761ff0..a312a87 100644
--- a/plugins/readers/deel.py
+++ b/plugins/readers/deel.py
@@ -14,17 +14,15 @@ read of the role's page.
from __future__ import annotations
-import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
-from ._shared import num, only, parse_dt, pay_line
+from ._shared import html_text, num, only, parse_dt, pay_line
log = logging.getLogger(__name__)
@@ -156,7 +154,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview",
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
"department": ", ".join(dict.fromkeys(departments)),
- "description": html.unescape(strip_html(description)),
+ "description": html_text(description),
"remote": arrangement or None,
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
pay.get("currencyIsoCode"), "")
diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py
index 0d776f7..3db4577 100644
--- a/plugins/readers/eightfold.py
+++ b/plugins/readers/eightfold.py
@@ -38,18 +38,16 @@ roles).
from __future__ import annotations
-import html
import json
import re
from collections.abc import Callable
from typing import Any
from urllib.parse import quote
-from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of
-from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
+from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
@@ -186,5 +184,4 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
data = answer.get("data") if isinstance(answer, dict) else None
if not isinstance(data, dict) or "jobDescription" not in data:
raise ValueError("not an Eightfold position: no data.jobDescription")
- text = html.unescape(strip_html(data["jobDescription"] or ""))
- return {"description": " ".join(text.split())}
+ return {"description": html_text(data["jobDescription"])}
diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py
index 1871833..696b5a0 100644
--- a/plugins/readers/gem.py
+++ b/plugins/readers/gem.py
@@ -17,17 +17,15 @@ The listing has no description and no date; both are on the role's page.
from __future__ import annotations
-import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import Get, Page, add_place, only, parse_dt
+from ._shared import Get, Page, add_place, html_text, only, parse_dt
log = logging.getLogger(__name__)
@@ -156,6 +154,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
if not isinstance(role, dict):
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
- text = html.unescape(strip_html(role.get("descriptionHtml") or ""))
- return {"description": " ".join(text.split()),
+ return {"description": html_text(role.get("descriptionHtml")),
"posted": parse_dt(role.get("firstPublishedTsSec"))}
diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py
index 7d39b80..e851b04 100644
--- a/plugins/readers/greenhouse.py
+++ b/plugins/readers/greenhouse.py
@@ -6,16 +6,14 @@ The field map is ported from `openings.go` in
from __future__ import annotations
-import html
import json
import re
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
+from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
NAME = "greenhouse"
LABEL = "Greenhouse"
@@ -154,10 +152,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
if not title or not is_web_url(url):
continue
departments = j.get("departments") or []
- # Greenhouse ships the description HTML-entity-escaped inside the
- # JSON string (`<p>`), so it needs unescaping before the tags
- # themselves can be stripped.
- content = html.unescape(j.get("content") or "")
ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges)
places = _places(j)
@@ -169,7 +163,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": url,
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
"department": (departments[0].get("name") if departments else "") or "",
- "description": strip_html(content),
+ # Greenhouse ships the description HTML-entity-escaped inside the
+ # JSON string (`<p>`), which `html_text` reads as markup.
+ "description": html_text(j.get("content")),
"remote": None, # Greenhouse's feed does not say
"salary_text": _salary_text(ranges),
"salary_min": salary_min,
diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py
index 2d65444..d87a4ab 100644
--- a/plugins/readers/hibob.py
+++ b/plugins/readers/hibob.py
@@ -14,16 +14,14 @@ filled, so the period's spelling is a guess.
from __future__ import annotations
-import html
import json
import logging
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
-from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
+from ._shared import Get, Page, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
log = logging.getLogger(__name__)
@@ -98,7 +96,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
country = str(ad.get("country") or "").strip()
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
else site or country
- text = " ".join(strip_html(str(ad.get(key) or "")) for key in _SECTIONS)
+ text = "\n".join(part for key in _SECTIONS
+ if (part := html_text(ad.get(key))))
lo, hi = num(ad.get("payTransparencyMinSalary")), num(ad.get("payTransparencyMaxSalary"))
currency = ad.get("payTransparencySalaryCurrency")
period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "")
@@ -114,7 +113,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"url": f"https://{_host(slug)}/jobs/{ad_id}",
"posted": parse_dt(ad.get("publishedAt")),
"department": str(ad.get("department") or "").strip(),
- "description": " ".join(html.unescape(text).split()),
+ "description": text,
"remote": str(ad.get("workspaceTypeId") or "").strip() or None,
"salary_text": pay_line(lo, hi, currency, period),
"salary_min": salary_min,
diff --git a/plugins/readers/hrmanager.py b/plugins/readers/hrmanager.py
index e5c4836..d1a50e0 100644
--- a/plugins/readers/hrmanager.py
+++ b/plugins/readers/hrmanager.py
@@ -23,16 +23,14 @@ The feed names no place (the title or the text may), so every row has
from __future__ import annotations
-import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
-from ._shared import Get, Page, is_web_url, only, parse_dt
+from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end",
@@ -118,7 +116,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
"url": url,
"posted": parse_dt(item.findtext("pubDate")),
"department": "",
- "description": html.unescape(strip_html(item.findtext("description") or "")),
+ "description": html_text(item.findtext("description")),
"remote": None,
"salary_text": "",
})
diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py
index 2e6d4f6..03f1a1e 100644
--- a/plugins/readers/jibe.py
+++ b/plugins/readers/jibe.py
@@ -28,17 +28,15 @@ any other site's, so the host is given by hand.
from __future__ import annotations
-import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import Get, Page, add_place, only, parse_dt, place
+from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
log = logging.getLogger(__name__)
@@ -103,10 +101,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
-def _plain(markup: Any) -> str:
- return " ".join(html.unescape(strip_html(str(markup or ""))).split())
-
-
def _places(j: dict[str, Any]) -> list[Place]:
"""`full_location` names every place of the role, "; " apart, each as
"City, Country"; the site's `city` and `country` stand in when it is
@@ -136,8 +130,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
if not title or not role:
continue
categories = j.get("categories") or []
- text = "\n\n".join(p for p in (_plain(j.get("description")),
- _plain(j.get("qualifications"))) if p)
+ text = "\n".join(p for p in (html_text(j.get("description")),
+ html_text(j.get("qualifications"))) if p)
places = _places(j)
row: Row = {
"source_id": role,
diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py
index cf063bb..477461c 100644
--- a/plugins/readers/jobylon.py
+++ b/plugins/readers/jobylon.py
@@ -26,7 +26,7 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Reads, Row
-from ._shared import Get, parse_dt, place
+from ._shared import Get, html_text, parse_dt, place
log = logging.getLogger(__name__)
@@ -227,7 +227,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if name and name not in places:
places.append(name)
found: dict[str, Any] = {
- "description": html.unescape(strip_html(data.get("description") or "")),
+ "description": html_text(data.get("description")),
"posted": parse_dt(data.get("datePosted")),
}
if places:
diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py
index 90256f2..7ee003b 100644
--- a/plugins/readers/lever.py
+++ b/plugins/readers/lever.py
@@ -20,10 +20,9 @@ import logging
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
-from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
+from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -137,7 +136,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": url,
"posted": parse_dt(j.get("createdAt")),
"department": (categories.get("department") or categories.get("team") or "").strip(),
- "description": strip_html(description),
+ "description": html_text(description),
"remote": (j.get("workplaceType") or "").strip() or None,
"salary_text": _salary_text(salary_range),
"salary_min": salary_min,
diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py
index c61b985..4762c82 100644
--- a/plugins/readers/oracle.py
+++ b/plugins/readers/oracle.py
@@ -39,7 +39,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of
-from ._shared import Get, Page, Row, add_place, only, parse_dt
+from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt
NAME = "oracle"
LABEL = "Oracle Recruiting Cloud"
@@ -231,9 +231,13 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if not items:
return {}
role = items[0]
- text = " ".join(part for key in ("ExternalDescriptionStr", "ExternalResponsibilitiesStr",
- "ExternalQualificationsStr")
- if (part := _plain(role.get(key))))
+ parts = []
+ for key, heading in (("ExternalDescriptionStr", ""),
+ ("ExternalResponsibilitiesStr", "Responsibilities"),
+ ("ExternalQualificationsStr", "Qualifications")):
+ if part := html_text(_text(role.get(key))):
+ parts += [HEADING + heading, part] if heading else [part]
+ text = "\n".join(parts)
out: dict[str, Any] = {}
if text:
out["description"] = text
diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py
index bb34b4d..070e4fd 100644
--- a/plugins/readers/personio.py
+++ b/plugins/readers/personio.py
@@ -10,10 +10,9 @@ from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Callable
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
-from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
+from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
NAME = "personio"
LABEL = "Personio"
@@ -128,7 +127,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
"posted": parse_dt(position.findtext("createdAt")),
"department": (position.findtext("department") or "").strip(),
- "description": strip_html(description),
+ "description": html_text(description),
"seniority": (position.findtext("seniority") or "").strip() or None,
"employment": _employment(position),
"salary_text": salary_text,
diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py
index 944255a..e49a101 100644
--- a/plugins/readers/phenom.py
+++ b/plugins/readers/phenom.py
@@ -35,7 +35,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import name_of
-from ._shared import Get, Page, Row, add_place, only, parse_dt
+from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "phenom"
LABEL = "Phenom"
@@ -243,8 +243,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
posting, job = _job_posting(body), _page_job(body)
out: dict[str, Any] = {}
- if description := (_plain(html.unescape(_text(posting.get("description"))))
- or _plain(job.get("description"))):
+ if description := (html_text(html.unescape(_text(posting.get("description"))))
+ or html_text(_text(job.get("description")))):
out["description"] = description
else:
raise ValueError("no description on the role's page")
diff --git a/plugins/readers/pinpoint.py b/plugins/readers/pinpoint.py
index f59fef9..cd4554c 100644
--- a/plugins/readers/pinpoint.py
+++ b/plugins/readers/pinpoint.py
@@ -16,15 +16,13 @@ so `posted` stays None rather than costing a request a role for it.
from __future__ import annotations
-import html
import json
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Reads
-from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
+from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
NAME = "pinpoint"
LABEL = "Pinpoint"
@@ -70,22 +68,17 @@ def feed_url(slug: str, ask: Ask = Ask()) -> str:
return f"https://{host(slug)}/postings.json"
-def _plain(markup: Any) -> str:
- """HTML as one line of plain text: the tags out, the entities read (Pinpoint
- writes `&` and ` `), the whitespace, non-breaking included, collapsed."""
- return " ".join(html.unescape(strip_html(str(markup or ""))).split())
-
-
def _text(j: dict[str, Any]) -> str:
"""The description and the sections after it, each section under its own
- heading, as plain text with a blank line between them."""
- parts = [_plain(j.get("description"))]
+ heading, as `html_text` lines."""
+ parts = [html_text(j.get("description"))]
for field in _SECTIONS:
- body = _plain(j.get(field))
+ body = html_text(j.get(field))
if body:
- heading = str(j.get(f"{field}_header") or "").strip()
- parts.append(f"{heading}: {body}" if heading else body)
- return "\n\n".join(p for p in parts if p)
+ if heading := " ".join(str(j.get(f"{field}_header") or "").split()):
+ parts.append(HEADING + heading)
+ parts.append(body)
+ return "\n".join(p for p in parts if p)
def _place(location: dict[str, Any]) -> str:
diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py
index 19e3183..3382dd1 100644
--- a/plugins/readers/platsbanken.py
+++ b/plugins/readers/platsbanken.py
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
-from ._shared import Get, Page, Row, is_web_url, only, parse_dt
+from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
NAME = "platsbanken"
LABEL = "Platsbanken"
@@ -124,6 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"country": "SE",
"remote": model or None,
"posted": parse_dt(hit.get("publication_date")),
- "description": _text((hit.get("description") or {}).get("text") or ""),
+ "description": html_text((hit.get("description") or {}).get("text")),
})
return out
diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py
index cce230b..e4c396c 100644
--- a/plugins/readers/recruitee.py
+++ b/plugins/readers/recruitee.py
@@ -11,10 +11,9 @@ import json
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
-from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
+from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
@@ -121,7 +120,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"url": url,
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
"department": (j.get("department") or "").strip(),
- "description": strip_html(j.get("description") or ""),
+ "description": html_text(j.get("description")),
"remote": j.get("remote"),
"salary_text": _salary_text(salary),
"salary_min": salary_min,
diff --git a/plugins/readers/rippling.py b/plugins/readers/rippling.py
index 1e98e4d..2104fb3 100644
--- a/plugins/readers/rippling.py
+++ b/plugins/readers/rippling.py
@@ -10,16 +10,14 @@ Board not found").
from __future__ import annotations
-import html
import json
import re
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
-from ._shared import Get, Row, add_place, parse_dt, yearly_amount
+from ._shared import Get, Row, add_place, html_text, parse_dt, yearly_amount
NAME = "rippling"
LABEL = "Rippling"
@@ -122,7 +120,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
data = json.loads(get(f"{_board(slug)}/jobs/{row['source_id']}"))
text = data.get("description") or {}
found: dict[str, Any] = {
- "description": html.unescape(strip_html(text.get("role") or text.get("company") or "")),
+ "description": html_text(text.get("role") or text.get("company")),
"posted": parse_dt(data.get("createdOn")),
}
if names := [p for p in (str(p).strip() for p in data.get("workLocations") or []) if p]:
diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py
index f8bb5a9..e433f0b 100644
--- a/plugins/readers/smartrecruiters.py
+++ b/plugins/readers/smartrecruiters.py
@@ -24,17 +24,15 @@ company blurb is the same on every role of the board and is left out.
from __future__ import annotations
-import html
import json
import re
from collections.abc import Callable
from typing import Any
from urllib.parse import quote
-from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
-from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
+from ._shared import Get, HEADING, Page, Row, html_text, only, parse_dt, path_slug_ok
NAME = "smartrecruiters"
LABEL = "SmartRecruiters"
@@ -162,8 +160,9 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
raise ValueError("not a SmartRecruiters posting: no jobAd.sections")
parts = []
for key in _SECTIONS:
- text = " ".join(html.unescape(strip_html(str((sections.get(key) or {}).get("text") or "")))
- .split())
- if text:
+ section = sections.get(key) or {}
+ if text := html_text(section.get("text")):
+ if title := " ".join(str(section.get("title") or "").split()):
+ parts.append(HEADING + title)
parts.append(text)
- return {"description": " ".join(parts)}
+ return {"description": "\n".join(parts)}
diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py
index 71c2f9a..03dfee2 100644
--- a/plugins/readers/successfactors.py
+++ b/plugins/readers/successfactors.py
@@ -63,7 +63,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
-from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
+from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
@@ -375,7 +375,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
# An employer's own layout can put a style block in the description.
block = re.sub(r"<(style|script)\b.*?\1>", " ", _element(text, match.start()),
flags=re.S | re.I)
- if description := _text(block):
+ if description := html_text(block):
out["description"] = description
if re.search(r'itemtype="https?://schema\.org/JobPosting"', text):
meta = {name: html.unescape(value) for name, value in _META.findall(text)}
diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py
index 328fc6c..cb4d7fc 100644
--- a/plugins/readers/teamtailor.py
+++ b/plugins/readers/teamtailor.py
@@ -8,16 +8,14 @@ Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
from __future__ import annotations
-import html
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
-from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
+from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
NAME = "teamtailor"
LABEL = "Teamtailor"
@@ -136,7 +134,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
"posted": parse_dt(item.findtext("pubDate")),
"department": ((item.findtext(f"{_TT}department") or "").strip()
or (item.findtext(f"{_TT}role") or "").strip()),
- "description": html.unescape(strip_html(item.findtext("description") or "")),
+ "description": html_text(item.findtext("description")),
"remote": status or None,
"salary_text": "", # Teamtailor's feed states no pay at all
})
diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py
index 36aa055..28b3d6e 100644
--- a/plugins/readers/workable.py
+++ b/plugins/readers/workable.py
@@ -15,15 +15,13 @@ reads and stops on one.
from __future__ import annotations
-import html
import json
from collections.abc import Callable
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
-from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
+from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt, path_slug_ok
NAME = "workable"
LABEL = "Workable"
@@ -146,7 +144,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
raise ValueError("not a Workable role: no description")
parts = []
for key in _PARTS:
- text = " ".join(html.unescape(strip_html(str(data.get(key) or ""))).split())
- if text:
+ if text := html_text(data.get(key)):
+ if key != "description":
+ parts.append(HEADING + key.capitalize())
parts.append(text)
- return {"description": " ".join(parts)}
+ return {"description": "\n".join(parts)}
diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py
index 02a253f..0cbebcf 100644
--- a/plugins/readers/workday.py
+++ b/plugins/readers/workday.py
@@ -31,18 +31,16 @@ A fetch term goes as `searchText` in the body, one a request
from __future__ import annotations
-import html
import json
import re
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any
-from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
-from ._shared import Get, Page, Row, add_place, only, parse_dt
+from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "workday"
LABEL = "Workday"
@@ -316,8 +314,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if not isinstance(posting, dict):
raise ValueError("not a Workday role: no jobPostingInfo")
out: dict[str, Any] = {}
- if description := " ".join(html.unescape(strip_html(posting.get("jobDescription") or ""))
- .split()):
+ if description := html_text(posting.get("jobDescription")):
out["description"] = description
remote = _remote(posting.get("remoteType"))
if places := _places(posting, remote):
diff --git a/tests/test_ats.py b/tests/test_ats.py
index 74d5b1a..e82808c 100644
--- a/tests/test_ats.py
+++ b/tests/test_ats.py
@@ -276,7 +276,7 @@ def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
first = rows[0]
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
assert first["department"] == "Engineering"
- assert first["description"] == "About the role Build & run care."
+ assert first["description"] == "## About the role\nBuild & run care."
assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC)
assert first["salary_text"] == ""
diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py
index 09f17d4..e2c27c6 100644
--- a/tests/test_ats_bamboohr.py
+++ b/tests/test_ats_bamboohr.py
@@ -113,7 +113,7 @@ def test_detail_of_the_standing_invitation() -> None:
url = "https://configit.bamboohr.com/careers/26/detail"
found = bamboohr.detail(_get_of({url: _read("bamboohr-detail-26.json")}, []), "configit", row)
assert found["posted"] == datetime(2025, 10, 10, tzinfo=UTC)
- assert found["description"].startswith("Hi, If there is currently no vacant position")
+ assert found["description"].startswith("Hi,\nIf there is currently no vacant position")
assert "UK (remote)" in found["description"]
diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py
index 7a313e4..38c230b 100644
--- a/tests/test_ats_deel.py
+++ b/tests/test_ats_deel.py
@@ -33,7 +33,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
"21584046-56d5-44c4-8b18-7e15dc659827/overview")
assert first["location"] == "Stockholm" and first["department"] == "Engineering"
assert first["remote"] == "ON_SITE"
- assert first["description"].startswith("Klarna, briefly At Klarna")
+ assert first["description"].startswith("## Klarna, briefly\nAt Klarna")
assert "<" not in first["description"] and "$" not in first["description"][:3]
assert all(r["description"] for r in rows)
# The day the posting was published, not the day it was drafted (12:27).
diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py
index 266098b..4d55f56 100644
--- a/tests/test_ats_eightfold.py
+++ b/tests/test_ats_eightfold.py
@@ -151,7 +151,7 @@ def test_detail_reads_the_text_of_the_ad() -> None:
"?position_id=563121777283313&domain=ericsson.com&hl=en"]
assert set(found) <= set(ats.DETAIL_FIELDS)
text = found["description"]
- assert text.startswith("Join our Team")
+ assert text.startswith("## Join our Team\n")
assert "<" not in text and "&" not in text and "\r" not in text and " " not in text
diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py
index c5f76bf..96579d0 100644
--- a/tests/test_ats_gem.py
+++ b/tests/test_ats_gem.py
@@ -129,7 +129,7 @@ def test_detail_reads_the_text_and_the_date() -> None:
assert asked[0]["query"] == gem.DETAIL_QUERY
assert set(found) <= set(ats.DETAIL_FIELDS)
assert found["posted"] == datetime.fromtimestamp(1787574069, tz=UTC)
- assert found["description"].startswith("Only the best engineers are being built here.")
+ assert found["description"].startswith("## Only the best engineers are being built here.\n")
assert "<" not in found["description"] and "&" not in found["description"]
diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py
index 1a1aa1e..3198948 100644
--- a/tests/test_ats_hrmanager.py
+++ b/tests/test_ats_hrmanager.py
@@ -36,7 +36,7 @@ def test_rows_read_the_advertisement_and_the_publication_date() -> None:
assert first["title"] == "Commissioning Engineer- T&D" # `&` in the XML
assert first["url"] == ("https://candidate.hr-manager.net/ApplicationInit.aspx"
"?cid=2523&ProjectId=143857&DepartmentId=18958&MediaId=5")
- assert first["description"].startswith("About the Position We have an exciting opportunity")
+ assert first["description"].startswith("## About the Position\nWe have an exciting opportunity")
assert "<" not in first["description"] and "&" not in first["description"]
assert "T&D Division" in first["description"]
assert first["posted"] is not None and first["posted"].isoformat() == \
diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py
index 05568c4..6de4180 100644
--- a/tests/test_ats_jibe.py
+++ b/tests/test_ats_jibe.py
@@ -64,7 +64,7 @@ def test_a_page_becomes_rows() -> None:
def test_qualifications_follow_the_description() -> None:
text = jibe.rows(_saved(), SITE)[-1]["description"]
- assert "\n\n" in text and text.split("\n\n")[-1].startswith("Java")
+ assert any(line.startswith("Java") for line in text.split("\n")[1:])
def test_the_location_falls_back_to_city_and_country() -> None:
diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py
index aba4256..f12d433 100644
--- a/tests/test_ats_oracle.py
+++ b/tests/test_ats_oracle.py
@@ -195,7 +195,7 @@ def test_detail_appends_the_responsibilities_and_qualifications_where_filled() -
data["items"][0]["ExternalResponsibilitiesStr"] = "
"
data["items"][0]["ExternalQualificationsStr"] = "
"}}}}).encode()
assert smartrecruiters.detail(lambda url, **_: body, "Acme", {"source_id": "7"}) == \
- {"description": "Build & run APIs"}
+ {"description": "Build & run\n\u2022 APIs"}
@pytest.mark.parametrize("body", [b"{}", b'{"jobAd": {}}', b'{"jobAd": null}', b"[]"])
diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py
index 9e0fb33..c3b4e4b 100644
--- a/tests/test_ats_successfactors.py
+++ b/tests/test_ats_successfactors.py
@@ -317,7 +317,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/"
def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None:
got = _detail("successfactors-job.html")
- assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you")
+ assert got["description"].startswith("## Senior Management | Full-Time | Commercial\nAre you")
assert "<" not in got["description"]
# `datePosted` is a Java date, "Tue Sep 29 02:00:00 UTC 2026".
assert got["posted"] == datetime(2026, 9, 29, 2, tzinfo=UTC)
@@ -347,14 +347,14 @@ def test_detail_reads_the_place_from_a_street_address() -> None:
got = _detail("successfactors-job-street.html")
assert got["location"] == "Bern, Switzerland"
assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC)
- assert got["description"].startswith("Senior Solution Engineer Defence")
+ assert got["description"].startswith("## Senior Solution Engineer Defence")
def test_detail_of_a_page_without_microdata_gives_the_description_only() -> None:
"""Novo Nordisk: the listing's date and location stand."""
got = _detail("successfactors-job-plain.html")
assert set(got) == {"description"}
- assert got["description"].startswith("Process Supporter, Koriyama Factory")
+ assert got["description"].startswith("## Process Supporter, Koriyama Factory\n")
assert "Your new role" in got["description"]
@@ -364,7 +364,7 @@ def test_detail_leaves_a_style_block_out_of_the_description() -> None:
'Nested
"}).encode()
assert workable.detail(lambda url, **_: body, "acme", {"shortcode": "AAA111"}) == \
- {"description": "Build & run Pension"}
+ {"description": "Build & run\n## Benefits\n\u2022 Pension"}
@pytest.mark.parametrize("body", [b"{}", b"[]", b'{"title": "x"}'])
diff --git a/tests/test_readers_text.py b/tests/test_readers_text.py
new file mode 100644
index 0000000..5b7b431
--- /dev/null
+++ b/tests/test_readers_text.py
@@ -0,0 +1,47 @@
+"""`html_text`: a host's description HTML as lines that keep its headings,
+bullets and paragraphs, and `excerpt` leaving those marks out of a row."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from app.models import excerpt # noqa: E402
+from plugins.readers._shared import html_text # noqa: E402
+
+
+def test_blocks_become_lines_lists_bullets_and_headings_marks() -> None:
+ markup = ("About
"
+ "
Line two