Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
17e7615599
commit
c497fe965d
44 files changed
+300
-147
No files matched your search
@@ -178,7 +178,7 @@ own id for the role, unique on the board), `title`, `url` (http or https,
|
||||
checked with `_shared.is_web_url`) and `places`; `location` (the host's
|
||||
one-line place, kept while storage reads it), `posted` (an aware datetime
|
||||
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
||||
see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
||||
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
||||
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||
|
||||
@@ -8,10 +8,12 @@ import it without either importing the other. The leading underscore keeps
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
@@ -216,3 +218,123 @@ def add_place(found: list[Place], entry: Place) -> None:
|
||||
found.append(entry)
|
||||
|
||||
|
||||
|
||||
|
||||
#: A description line that is a heading, and one that is a list item; what
|
||||
#: the reader pane draws as `<h3>` and `<li>`. The rest of a description is
|
||||
#: plain text, one paragraph a line.
|
||||
HEADING = "## "
|
||||
BULLET = "\u2022 "
|
||||
|
||||
#: Tags that end a line. A `<li>` starts a bullet line, an `<h1>`..`<h6>` a
|
||||
#: heading line; the rest start a paragraph.
|
||||
_BLOCKS = frozenset({
|
||||
"address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "fieldset",
|
||||
"figcaption", "figure", "footer", "form", "header", "hr", "li", "main", "nav", "ol", "p",
|
||||
"pre", "section", "table", "tbody", "thead", "tfoot", "tr", "ul",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6",
|
||||
})
|
||||
_HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
|
||||
_BOLD = frozenset({"b", "strong"})
|
||||
_SKIPPED = frozenset({"script", "style", "head", "template", "noscript"})
|
||||
#: A paragraph set wholly in bold is a heading when it is at most this long;
|
||||
#: longer, it is a paragraph somebody made loud.
|
||||
_BOLD_HEADING = 80
|
||||
_TAG = re.compile(r"</?[A-Za-z][^>]*>")
|
||||
_ESCAPED_TAG = re.compile(r"</?[A-Za-z]")
|
||||
_ENTITY = re.compile(r"&(#\d+|#x[0-9A-Fa-f]+|[A-Za-z]+);")
|
||||
#: A bullet typed as text: `- item`, `* item`, `\u00b7 item` and the like.
|
||||
_TYPED_BULLET = re.compile("^[\u2022\u00b7\u25aa\u25cf\u25e6\u2023\u2043*\\-\u2013]\\s+")
|
||||
|
||||
|
||||
class _Lines(HTMLParser):
|
||||
"""`html_text`'s parser: a line per block, each a kind ("p", "li" or
|
||||
"h"), its text, and whether all of the text was bold."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.lines: list[tuple[str, str, bool]] = []
|
||||
self.kind = "p"
|
||||
self.pieces: list[tuple[str, bool]] = []
|
||||
self.bold = 0
|
||||
self.skip = 0
|
||||
|
||||
def has_text(self) -> bool:
|
||||
return any(text.strip() for text, _ in self.pieces)
|
||||
|
||||
def flush(self) -> None:
|
||||
if self.has_text():
|
||||
text = "".join(text for text, _ in self.pieces)
|
||||
bold = all(b for text, b in self.pieces if text.strip())
|
||||
self.lines.append((self.kind, text, bold))
|
||||
self.pieces = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag in _SKIPPED:
|
||||
self.skip += 1
|
||||
elif tag in _BOLD:
|
||||
self.bold += 1
|
||||
elif tag in ("td", "th"):
|
||||
self.pieces.append((" ", True))
|
||||
elif tag in _BLOCKS:
|
||||
kind = "li" if tag == "li" else "h" if tag in _HEADINGS else ""
|
||||
if self.has_text():
|
||||
self.flush()
|
||||
self.kind = kind or "p"
|
||||
elif kind:
|
||||
self.kind = kind
|
||||
# else `<li><p>text</p></li>`: the `<p>` keeps the bullet.
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in _SKIPPED:
|
||||
self.skip = max(0, self.skip - 1)
|
||||
elif tag in _BOLD:
|
||||
self.bold = max(0, self.bold - 1)
|
||||
elif tag in _BLOCKS:
|
||||
self.flush()
|
||||
self.kind = "p"
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if not self.skip:
|
||||
self.pieces.append((data, self.bold > 0))
|
||||
|
||||
|
||||
def _line(kind: str, text: str, bold: bool) -> str:
|
||||
text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\ufeff", "")
|
||||
if _ENTITY.search(text):
|
||||
# Escaped twice at the host (`&amp;`): once is left after parsing.
|
||||
text = html.unescape(text)
|
||||
text = " ".join(text.split())
|
||||
if typed := _TYPED_BULLET.match(text):
|
||||
text, kind = text[typed.end():], "li"
|
||||
if not text or text in ("\u2022", "\u00b7", "-", "*"):
|
||||
return ""
|
||||
if kind == "li":
|
||||
return BULLET + text
|
||||
if kind == "h" or (bold and len(text) <= _BOLD_HEADING):
|
||||
return HEADING + text
|
||||
return text
|
||||
|
||||
|
||||
def html_text(markup: Any) -> str:
|
||||
"""A host's description HTML as plain text that keeps its shape: a line
|
||||
per paragraph, `BULLET` before a list item and `HEADING` before a
|
||||
heading (an `<h1>`..`<h6>`, or a short paragraph set wholly in bold).
|
||||
Entities are decoded, and markup the host escaped once more
|
||||
(`<p>`) is read as markup. Text with no tags keeps its own line
|
||||
breaks, and a line typed as a bullet (`- item`) becomes one.
|
||||
|
||||
This is a row's `description`: what the reader pane draws as headings,
|
||||
lists and paragraphs, and what summarise and embed read. A one-line
|
||||
field (a title, a teaser) stays `desk.sources.strip_html`."""
|
||||
markup = str(markup or "")
|
||||
if not _TAG.search(markup):
|
||||
if _ESCAPED_TAG.search(markup):
|
||||
markup = html.unescape(markup)
|
||||
else:
|
||||
markup = "<br>".join(html.escape(line, quote=False) for line in markup.splitlines())
|
||||
parser = _Lines()
|
||||
parser.feed(markup)
|
||||
parser.close()
|
||||
parser.flush()
|
||||
return "\n".join(line for line in (_line(*entry) for entry in parser.lines) if line)
|
||||
@@ -12,10 +12,9 @@ import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -142,7 +141,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("publishedAt")),
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"description": html_text(description),
|
||||
"remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too
|
||||
"salary_text": _salary_text(compensation),
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -14,16 +14,14 @@ first) is listed like any other role.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, label_slug_ok, only, parse_dt, place
|
||||
from ._shared import Get, html_text, label_slug_ok, only, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -141,7 +139,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if not isinstance(opening, dict):
|
||||
raise ValueError("not a BambooHR role: no result.jobOpening")
|
||||
out: dict[str, Any] = {
|
||||
"description": " ".join(html.unescape(strip_html(opening.get("description") or "")).split()),
|
||||
"description": html_text(opening.get("description")),
|
||||
"posted": parse_dt(opening.get("datePosted")),
|
||||
}
|
||||
if place := _place(opening):
|
||||
|
||||
@@ -14,17 +14,15 @@ read of the role's page.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import num, only, parse_dt, pay_line
|
||||
from ._shared import html_text, num, only, parse_dt, pay_line
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -156,7 +154,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview",
|
||||
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
|
||||
"department": ", ".join(dict.fromkeys(departments)),
|
||||
"description": html.unescape(strip_html(description)),
|
||||
"description": html_text(description),
|
||||
"remote": arrangement or None,
|
||||
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
|
||||
pay.get("currencyIsoCode"), "")
|
||||
|
||||
@@ -38,18 +38,16 @@ roles).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
||||
from ._countries import name_of
|
||||
from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
|
||||
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||
@@ -186,5 +184,4 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
data = answer.get("data") if isinstance(answer, dict) else None
|
||||
if not isinstance(data, dict) or "jobDescription" not in data:
|
||||
raise ValueError("not an Eightfold position: no data.jobDescription")
|
||||
text = html.unescape(strip_html(data["jobDescription"] or ""))
|
||||
return {"description": " ".join(text.split())}
|
||||
return {"description": html_text(data["jobDescription"])}
|
||||
@@ -17,17 +17,15 @@ The listing has no description and no date; both are on the role's page.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, add_place, only, parse_dt
|
||||
from ._shared import Get, Page, add_place, html_text, only, parse_dt
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -156,6 +154,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
|
||||
if not isinstance(role, dict):
|
||||
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
|
||||
text = html.unescape(strip_html(role.get("descriptionHtml") or ""))
|
||||
return {"description": " ".join(text.split()),
|
||||
return {"description": html_text(role.get("descriptionHtml")),
|
||||
"posted": parse_dt(role.get("firstPublishedTsSec"))}
|
||||
@@ -6,16 +6,14 @@ The field map is ported from `openings.go` in
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||
|
||||
NAME = "greenhouse"
|
||||
LABEL = "Greenhouse"
|
||||
@@ -154,10 +152,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
departments = j.get("departments") or []
|
||||
# Greenhouse ships the description HTML-entity-escaped inside the
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
ranges = j.get("pay_input_ranges") or []
|
||||
salary_min, salary_max, currency = _salary(ranges)
|
||||
places = _places(j)
|
||||
@@ -169,7 +163,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
|
||||
"department": (departments[0].get("name") if departments else "") or "",
|
||||
"description": strip_html(content),
|
||||
# Greenhouse ships the description HTML-entity-escaped inside the
|
||||
# JSON string (`<p>`), which `html_text` reads as markup.
|
||||
"description": html_text(j.get("content")),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary_text": _salary_text(ranges),
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -14,16 +14,14 @@ filled, so the period's spelling is a guess.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Get, Page, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -98,7 +96,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
country = str(ad.get("country") or "").strip()
|
||||
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
||||
else site or country
|
||||
text = " ".join(strip_html(str(ad.get(key) or "")) for key in _SECTIONS)
|
||||
text = "\n".join(part for key in _SECTIONS
|
||||
if (part := html_text(ad.get(key))))
|
||||
lo, hi = num(ad.get("payTransparencyMinSalary")), num(ad.get("payTransparencyMaxSalary"))
|
||||
currency = ad.get("payTransparencySalaryCurrency")
|
||||
period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "")
|
||||
@@ -114,7 +113,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"url": f"https://{_host(slug)}/jobs/{ad_id}",
|
||||
"posted": parse_dt(ad.get("publishedAt")),
|
||||
"department": str(ad.get("department") or "").strip(),
|
||||
"description": " ".join(html.unescape(text).split()),
|
||||
"description": text,
|
||||
"remote": str(ad.get("workspaceTypeId") or "").strip() or None,
|
||||
"salary_text": pay_line(lo, hi, currency, period),
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -23,16 +23,14 @@ The feed names no place (the title or the text may), so every row has
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, Page, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
||||
@@ -118,7 +116,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"url": url,
|
||||
"posted": parse_dt(item.findtext("pubDate")),
|
||||
"department": "",
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"description": html_text(item.findtext("description")),
|
||||
"remote": None,
|
||||
"salary_text": "",
|
||||
})
|
||||
|
||||
@@ -28,17 +28,15 @@ any other site's, so the host is given by hand.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, place
|
||||
from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -103,10 +101,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _plain(markup: Any) -> str:
|
||||
return " ".join(html.unescape(strip_html(str(markup or ""))).split())
|
||||
|
||||
|
||||
def _places(j: dict[str, Any]) -> list[Place]:
|
||||
"""`full_location` names every place of the role, "; " apart, each as
|
||||
"City, Country"; the site's `city` and `country` stand in when it is
|
||||
@@ -136,8 +130,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
if not title or not role:
|
||||
continue
|
||||
categories = j.get("categories") or []
|
||||
text = "\n\n".join(p for p in (_plain(j.get("description")),
|
||||
_plain(j.get("qualifications"))) if p)
|
||||
text = "\n".join(p for p in (html_text(j.get("description")),
|
||||
html_text(j.get("qualifications"))) if p)
|
||||
places = _places(j)
|
||||
row: Row = {
|
||||
"source_id": role,
|
||||
|
||||
@@ -26,7 +26,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, parse_dt, place
|
||||
from ._shared import Get, html_text, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -227,7 +227,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if name and name not in places:
|
||||
places.append(name)
|
||||
found: dict[str, Any] = {
|
||||
"description": html.unescape(strip_html(data.get("description") or "")),
|
||||
"description": html_text(data.get("description")),
|
||||
"posted": parse_dt(data.get("datePosted")),
|
||||
}
|
||||
if places:
|
||||
|
||||
@@ -20,10 +20,9 @@ import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -137,7 +136,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("createdAt")),
|
||||
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"description": html_text(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary_text": _salary_text(salary_range),
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -39,7 +39,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES, code_of
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt
|
||||
|
||||
NAME = "oracle"
|
||||
LABEL = "Oracle Recruiting Cloud"
|
||||
@@ -231,9 +231,13 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if not items:
|
||||
return {}
|
||||
role = items[0]
|
||||
text = " ".join(part for key in ("ExternalDescriptionStr", "ExternalResponsibilitiesStr",
|
||||
"ExternalQualificationsStr")
|
||||
if (part := _plain(role.get(key))))
|
||||
parts = []
|
||||
for key, heading in (("ExternalDescriptionStr", ""),
|
||||
("ExternalResponsibilitiesStr", "Responsibilities"),
|
||||
("ExternalQualificationsStr", "Qualifications")):
|
||||
if part := html_text(_text(role.get(key))):
|
||||
parts += [HEADING + heading, part] if heading else [part]
|
||||
text = "\n".join(parts)
|
||||
out: dict[str, Any] = {}
|
||||
if text:
|
||||
out["description"] = text
|
||||
|
||||
@@ -10,10 +10,9 @@ from __future__ import annotations
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
|
||||
from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
|
||||
|
||||
NAME = "personio"
|
||||
LABEL = "Personio"
|
||||
@@ -128,7 +127,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
||||
"posted": parse_dt(position.findtext("createdAt")),
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"description": html_text(description),
|
||||
"seniority": (position.findtext("seniority") or "").strip() or None,
|
||||
"employment": _employment(position),
|
||||
"salary_text": salary_text,
|
||||
|
||||
@@ -35,7 +35,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import name_of
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||
|
||||
NAME = "phenom"
|
||||
LABEL = "Phenom"
|
||||
@@ -243,8 +243,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
|
||||
posting, job = _job_posting(body), _page_job(body)
|
||||
out: dict[str, Any] = {}
|
||||
if description := (_plain(html.unescape(_text(posting.get("description"))))
|
||||
or _plain(job.get("description"))):
|
||||
if description := (html_text(html.unescape(_text(posting.get("description"))))
|
||||
or html_text(_text(job.get("description")))):
|
||||
out["description"] = description
|
||||
else:
|
||||
raise ValueError("no description on the role's page")
|
||||
|
||||
@@ -16,15 +16,13 @@ so `posted` stays None rather than costing a request a role for it.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
|
||||
NAME = "pinpoint"
|
||||
LABEL = "Pinpoint"
|
||||
@@ -70,22 +68,17 @@ def feed_url(slug: str, ask: Ask = Ask()) -> str:
|
||||
return f"https://{host(slug)}/postings.json"
|
||||
|
||||
|
||||
def _plain(markup: Any) -> str:
|
||||
"""HTML as one line of plain text: the tags out, the entities read (Pinpoint
|
||||
writes `&` and ` `), the whitespace, non-breaking included, collapsed."""
|
||||
return " ".join(html.unescape(strip_html(str(markup or ""))).split())
|
||||
|
||||
|
||||
def _text(j: dict[str, Any]) -> str:
|
||||
"""The description and the sections after it, each section under its own
|
||||
heading, as plain text with a blank line between them."""
|
||||
parts = [_plain(j.get("description"))]
|
||||
heading, as `html_text` lines."""
|
||||
parts = [html_text(j.get("description"))]
|
||||
for field in _SECTIONS:
|
||||
body = _plain(j.get(field))
|
||||
body = html_text(j.get(field))
|
||||
if body:
|
||||
heading = str(j.get(f"{field}_header") or "").strip()
|
||||
parts.append(f"{heading}: {body}" if heading else body)
|
||||
return "\n\n".join(p for p in parts if p)
|
||||
if heading := " ".join(str(j.get(f"{field}_header") or "").split()):
|
||||
parts.append(HEADING + heading)
|
||||
parts.append(body)
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _place(location: dict[str, Any]) -> str:
|
||||
|
||||
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "platsbanken"
|
||||
LABEL = "Platsbanken"
|
||||
@@ -124,6 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
"country": "SE",
|
||||
"remote": model or None,
|
||||
"posted": parse_dt(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
"description": html_text((hit.get("description") or {}).get("text")),
|
||||
})
|
||||
return out
|
||||
@@ -11,10 +11,9 @@ import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "recruitee"
|
||||
LABEL = "Recruitee"
|
||||
@@ -121,7 +120,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"description": html_text(j.get("description")),
|
||||
"remote": j.get("remote"),
|
||||
"salary_text": _salary_text(salary),
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -10,16 +10,14 @@ Board not found").
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
|
||||
from ._shared import Get, Row, add_place, html_text, parse_dt, yearly_amount
|
||||
|
||||
NAME = "rippling"
|
||||
LABEL = "Rippling"
|
||||
@@ -122,7 +120,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
data = json.loads(get(f"{_board(slug)}/jobs/{row['source_id']}"))
|
||||
text = data.get("description") or {}
|
||||
found: dict[str, Any] = {
|
||||
"description": html.unescape(strip_html(text.get("role") or text.get("company") or "")),
|
||||
"description": html_text(text.get("role") or text.get("company")),
|
||||
"posted": parse_dt(data.get("createdOn")),
|
||||
}
|
||||
if names := [p for p in (str(p).strip() for p in data.get("workLocations") or []) if p]:
|
||||
|
||||
@@ -24,17 +24,15 @@ company blurb is the same on every role of the board and is left out.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
from ._shared import Get, HEADING, Page, Row, html_text, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "smartrecruiters"
|
||||
LABEL = "SmartRecruiters"
|
||||
@@ -162,8 +160,9 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
raise ValueError("not a SmartRecruiters posting: no jobAd.sections")
|
||||
parts = []
|
||||
for key in _SECTIONS:
|
||||
text = " ".join(html.unescape(strip_html(str((sections.get(key) or {}).get("text") or "")))
|
||||
.split())
|
||||
if text:
|
||||
section = sections.get(key) or {}
|
||||
if text := html_text(section.get("text")):
|
||||
if title := " ".join(str(section.get("title") or "").split()):
|
||||
parts.append(HEADING + title)
|
||||
parts.append(text)
|
||||
return {"description": " ".join(parts)}
|
||||
return {"description": "\n".join(parts)}
|
||||
@@ -63,7 +63,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES
|
||||
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "successfactors"
|
||||
LABEL = "SuccessFactors"
|
||||
@@ -375,7 +375,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
# An employer's own layout can put a style block in the description.
|
||||
block = re.sub(r"<(style|script)\b.*?</\1>", " ", _element(text, match.start()),
|
||||
flags=re.S | re.I)
|
||||
if description := _text(block):
|
||||
if description := html_text(block):
|
||||
out["description"] = description
|
||||
if re.search(r'itemtype="https?://schema\.org/JobPosting"', text):
|
||||
meta = {name: html.unescape(value) for name, value in _META.findall(text)}
|
||||
|
||||
@@ -8,16 +8,14 @@ Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "teamtailor"
|
||||
LABEL = "Teamtailor"
|
||||
@@ -136,7 +134,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"posted": parse_dt(item.findtext("pubDate")),
|
||||
"department": ((item.findtext(f"{_TT}department") or "").strip()
|
||||
or (item.findtext(f"{_TT}role") or "").strip()),
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"description": html_text(item.findtext("description")),
|
||||
"remote": status or None,
|
||||
"salary_text": "", # Teamtailor's feed states no pay at all
|
||||
})
|
||||
|
||||
@@ -15,15 +15,13 @@ reads and stops on one.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
|
||||
from ._shared import Get, HEADING, Page, Row, add_place, html_text, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "workable"
|
||||
LABEL = "Workable"
|
||||
@@ -146,7 +144,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
raise ValueError("not a Workable role: no description")
|
||||
parts = []
|
||||
for key in _PARTS:
|
||||
text = " ".join(html.unescape(strip_html(str(data.get(key) or ""))).split())
|
||||
if text:
|
||||
if text := html_text(data.get(key)):
|
||||
if key != "description":
|
||||
parts.append(HEADING + key.capitalize())
|
||||
parts.append(text)
|
||||
return {"description": " ".join(parts)}
|
||||
return {"description": "\n".join(parts)}
|
||||
@@ -31,18 +31,16 @@ A fetch term goes as `searchText` in the body, one a request
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import code_of
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
@@ -316,8 +314,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if not isinstance(posting, dict):
|
||||
raise ValueError("not a Workday role: no jobPostingInfo")
|
||||
out: dict[str, Any] = {}
|
||||
if description := " ".join(html.unescape(strip_html(posting.get("jobDescription") or ""))
|
||||
.split()):
|
||||
if description := html_text(posting.get("jobDescription")):
|
||||
out["description"] = description
|
||||
remote = _remote(posting.get("remoteType"))
|
||||
if places := _places(posting, remote):
|
||||
|
||||
Reference in new issue
Block a user