Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
17e7615599
commit
c497fe965d
44 files changed
+300
-147
No files matched your search
+1
-1
@@ -276,7 +276,7 @@ def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
||||
first = rows[0]
|
||||
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
||||
assert first["department"] == "Engineering"
|
||||
assert first["description"] == "About the role Build & run care."
|
||||
assert first["description"] == "## About the role\nBuild & run care."
|
||||
assert first["posted"] == datetime(2026, 9, 23, 10, 59, 53, tzinfo=UTC)
|
||||
assert first["salary_text"] == ""
|
||||
|
||||
|
||||
@@ -113,7 +113,7 @@ def test_detail_of_the_standing_invitation() -> None:
|
||||
url = "https://configit.bamboohr.com/careers/26/detail"
|
||||
found = bamboohr.detail(_get_of({url: _read("bamboohr-detail-26.json")}, []), "configit", row)
|
||||
assert found["posted"] == datetime(2025, 10, 10, tzinfo=UTC)
|
||||
assert found["description"].startswith("Hi, If there is currently no vacant position")
|
||||
assert found["description"].startswith("Hi,\nIf there is currently no vacant position")
|
||||
assert "UK (remote)" in found["description"]
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
|
||||
"21584046-56d5-44c4-8b18-7e15dc659827/overview")
|
||||
assert first["location"] == "Stockholm" and first["department"] == "Engineering"
|
||||
assert first["remote"] == "ON_SITE"
|
||||
assert first["description"].startswith("Klarna, briefly At Klarna")
|
||||
assert first["description"].startswith("## Klarna, briefly\nAt Klarna")
|
||||
assert "<" not in first["description"] and "$" not in first["description"][:3]
|
||||
assert all(r["description"] for r in rows)
|
||||
# The day the posting was published, not the day it was drafted (12:27).
|
||||
|
||||
@@ -151,7 +151,7 @@ def test_detail_reads_the_text_of_the_ad() -> None:
|
||||
"?position_id=563121777283313&domain=ericsson.com&hl=en"]
|
||||
assert set(found) <= set(ats.DETAIL_FIELDS)
|
||||
text = found["description"]
|
||||
assert text.startswith("Join our Team")
|
||||
assert text.startswith("## Join our Team\n")
|
||||
assert "<" not in text and "&" not in text and "\r" not in text and " " not in text
|
||||
|
||||
|
||||
|
||||
@@ -129,7 +129,7 @@ def test_detail_reads_the_text_and_the_date() -> None:
|
||||
assert asked[0]["query"] == gem.DETAIL_QUERY
|
||||
assert set(found) <= set(ats.DETAIL_FIELDS)
|
||||
assert found["posted"] == datetime.fromtimestamp(1787574069, tz=UTC)
|
||||
assert found["description"].startswith("Only the best engineers are being built here.")
|
||||
assert found["description"].startswith("## Only the best engineers are being built here.\n")
|
||||
assert "<" not in found["description"] and "&" not in found["description"]
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_rows_read_the_advertisement_and_the_publication_date() -> None:
|
||||
assert first["title"] == "Commissioning Engineer- T&D" # `&` in the XML
|
||||
assert first["url"] == ("https://candidate.hr-manager.net/ApplicationInit.aspx"
|
||||
"?cid=2523&ProjectId=143857&DepartmentId=18958&MediaId=5")
|
||||
assert first["description"].startswith("About the Position We have an exciting opportunity")
|
||||
assert first["description"].startswith("## About the Position\nWe have an exciting opportunity")
|
||||
assert "<" not in first["description"] and "&" not in first["description"]
|
||||
assert "T&D Division" in first["description"]
|
||||
assert first["posted"] is not None and first["posted"].isoformat() == \
|
||||
|
||||
@@ -64,7 +64,7 @@ def test_a_page_becomes_rows() -> None:
|
||||
|
||||
def test_qualifications_follow_the_description() -> None:
|
||||
text = jibe.rows(_saved(), SITE)[-1]["description"]
|
||||
assert "\n\n" in text and text.split("\n\n")[-1].startswith("Java")
|
||||
assert any(line.startswith("Java") for line in text.split("\n")[1:])
|
||||
|
||||
|
||||
def test_the_location_falls_back_to_city_and_country() -> None:
|
||||
|
||||
@@ -195,7 +195,7 @@ def test_detail_appends_the_responsibilities_and_qualifications_where_filled() -
|
||||
data["items"][0]["ExternalResponsibilitiesStr"] = "<ul><li>Run the desk</li></ul>"
|
||||
data["items"][0]["ExternalQualificationsStr"] = "<p>Five years & more</p>"
|
||||
got = oracle.detail(_detail_get(json.dumps(data).encode(), []), SLUG, {"source_id": "24594"})
|
||||
assert got["description"].endswith(" Run the desk Five years & more")
|
||||
assert got["description"].endswith("\n## Responsibilities\n\u2022 Run the desk\n## Qualifications\nFive years & more")
|
||||
|
||||
|
||||
def test_detail_of_a_role_no_longer_open_has_nothing_to_read() -> None:
|
||||
|
||||
@@ -205,7 +205,7 @@ def test_detail_reads_the_description_from_the_json_ld() -> None:
|
||||
row = _rows()[0]
|
||||
got = phenom.detail(_get_of("phenom-detail.html", asked), SLUG, row)
|
||||
assert asked == [(row["url"], {"Accept": "text/html"})]
|
||||
assert got["description"].startswith("Craft: Data, Analytics & Strategy")
|
||||
assert got["description"].startswith("## Craft:\nData, Analytics & Strategy")
|
||||
assert "<" not in got["description"] and "<" not in got["description"]
|
||||
assert "&" not in got["description"]
|
||||
assert got["posted"].isoformat() == "2026-09-28T00:00:00+00:00"
|
||||
@@ -215,7 +215,7 @@ def test_detail_reads_the_description_from_the_json_ld() -> None:
|
||||
def test_detail_finds_the_job_posting_among_other_blocks_and_takes_its_date() -> None:
|
||||
got = phenom.detail(_get_of("phenom-detail-2.html", []), "www.careers.philips.com/global/en",
|
||||
{"url": "https://www.careers.philips.com/global/en/job/1/x"})
|
||||
assert got["description"].startswith("Job Title Zone Business Leader Imaging Nordics")
|
||||
assert got["description"].startswith("## Job Title\nZone Business Leader Imaging Nordics")
|
||||
assert got["posted"].isoformat() == "2026-09-08T00:00:00+00:00"
|
||||
|
||||
|
||||
@@ -224,7 +224,7 @@ def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() ->
|
||||
`WebPage` one, and the role in the state the page assigns to `phApp.ddo`."""
|
||||
row = _rows()[0]
|
||||
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
|
||||
assert got["description"].startswith("Craft: Games Design & Production Job Description:")
|
||||
assert got["description"].startswith("## Craft:\nGames Design & Production\n## Job Description:")
|
||||
assert "<" not in got["description"] and "&" not in got["description"]
|
||||
assert got["remote"] == "On-site"
|
||||
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
|
||||
|
||||
@@ -39,13 +39,13 @@ def test_a_listing_becomes_rows() -> None:
|
||||
first["currency"]) == ("", None, None, "")
|
||||
|
||||
|
||||
def test_the_description_joins_the_sections_as_plain_text() -> None:
|
||||
def test_the_description_puts_each_section_under_its_heading() -> None:
|
||||
text = pinpoint.parse(_body(), "xeneta")[0]["description"]
|
||||
assert text.startswith("Data Detective Wanted to join our Data & Analytics Team")
|
||||
assert text.startswith("## Data Detective Wanted to join our Data & Analytics Team\n")
|
||||
assert "<" not in text and "&" not in text and "\xa0" not in text
|
||||
assert "\n\nKey Responsibilities: Join our Liner Data & Analytics Team" in text
|
||||
assert "\n\nSkills, Knowledge & Expertise: In this role" in text
|
||||
assert "\n\nJob Benefits: We will help you grow your career:" in text
|
||||
assert "\n## Key Responsibilities\nJoin our Liner Data & Analytics Team" in text
|
||||
assert "\n## Skills, Knowledge & Expertise\nIn this role" in text
|
||||
assert "\n## Job Benefits\n\u2022 We will help you grow your career:" in text
|
||||
|
||||
|
||||
def test_a_region_is_kept_when_it_says_more_than_the_name() -> None:
|
||||
|
||||
@@ -85,7 +85,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked),
|
||||
"rippling", row)
|
||||
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
|
||||
assert found["description"].startswith("About The Role")
|
||||
assert found["description"].startswith("## About The Role\n")
|
||||
assert "<" not in found["description"]
|
||||
assert found["location"] == "San Francisco, CA; Seattle, WA; New York, NY"
|
||||
assert [p["text"] for p in found["places"]] == ["San Francisco, CA", "Seattle, WA", "New York, NY"]
|
||||
@@ -98,7 +98,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
|
||||
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()),
|
||||
"prisma-careers", {"source_id": "x"})
|
||||
assert found["description"].startswith("What is it like to work at Prisma?")
|
||||
assert found["description"].startswith("## What is it like to work at Prisma?\n")
|
||||
assert found["location"] == "Remote (Germany)"
|
||||
assert found["places"] == [{"text": "Remote (Germany)", "remote": True}]
|
||||
assert not {"salary_min", "salary_max", "currency"} & set(found)
|
||||
|
||||
@@ -103,7 +103,7 @@ def test_detail_leaves_out_a_section_the_ad_lacks() -> None:
|
||||
body = json.dumps({"jobAd": {"sections": {
|
||||
"jobDescription": {"text": "<p>Build & run</p><ul><li>APIs</li></ul>"}}}}).encode()
|
||||
assert smartrecruiters.detail(lambda url, **_: body, "Acme", {"source_id": "7"}) == \
|
||||
{"description": "Build & run APIs"}
|
||||
{"description": "Build & run\n\u2022 APIs"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"{}", b'{"jobAd": {}}', b'{"jobAd": null}', b"[]"])
|
||||
|
||||
@@ -317,7 +317,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/"
|
||||
|
||||
def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None:
|
||||
got = _detail("successfactors-job.html")
|
||||
assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you")
|
||||
assert got["description"].startswith("## Senior Management | Full-Time | Commercial\nAre you")
|
||||
assert "<" not in got["description"]
|
||||
# `datePosted` is a Java date, "Tue Sep 29 02:00:00 UTC 2026".
|
||||
assert got["posted"] == datetime(2026, 9, 29, 2, tzinfo=UTC)
|
||||
@@ -347,14 +347,14 @@ def test_detail_reads_the_place_from_a_street_address() -> None:
|
||||
got = _detail("successfactors-job-street.html")
|
||||
assert got["location"] == "Bern, Switzerland"
|
||||
assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC)
|
||||
assert got["description"].startswith("Senior Solution Engineer Defence")
|
||||
assert got["description"].startswith("## Senior Solution Engineer Defence")
|
||||
|
||||
|
||||
def test_detail_of_a_page_without_microdata_gives_the_description_only() -> None:
|
||||
"""Novo Nordisk: the listing's date and location stand."""
|
||||
got = _detail("successfactors-job-plain.html")
|
||||
assert set(got) == {"description"}
|
||||
assert got["description"].startswith("Process Supporter, Koriyama Factory")
|
||||
assert got["description"].startswith("## Process Supporter, Koriyama Factory\n")
|
||||
assert "Your new role" in got["description"]
|
||||
|
||||
|
||||
@@ -364,7 +364,7 @@ def test_detail_leaves_a_style_block_out_of_the_description() -> None:
|
||||
'<span>Nested</span></span></span></div><p>Footer</p>')
|
||||
got = successfactors.detail(lambda url, **kw: page.encode(), "careers.acme.com",
|
||||
{"url": "https://careers.acme.com/job/x/1/"})
|
||||
assert got == {"description": "Build things. Nested"}
|
||||
assert got == {"description": "Build things.\nNested"}
|
||||
|
||||
|
||||
def test_detail_of_a_page_that_is_not_a_role_is_empty() -> None:
|
||||
@@ -385,7 +385,7 @@ def test_detail_is_read_through_the_shared_code_for_the_roles_not_read(
|
||||
assert done == {f"successfactors:careers.vestas.com:{r['source_id']}" for r in rows}
|
||||
assert not failed
|
||||
assert out[0]["location"] == "Seoul, South Korea"
|
||||
assert out[0]["description"].startswith("Senior Management")
|
||||
assert out[0]["description"].startswith("## Senior Management")
|
||||
assert out[0]["title"] == rows[0]["title"]
|
||||
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.sources import strip_html # noqa: E402
|
||||
from plugins.readers import Ask, _runner, workable # noqa: E402
|
||||
from plugins.readers._shared import html_text # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -86,9 +86,8 @@ def test_detail_reads_description_requirements_and_benefits() -> None:
|
||||
assert set(found) == {"description"} and set(found) <= set(_runner.DETAIL_FIELDS)
|
||||
text, data = found["description"], json.loads(body)
|
||||
for key in ("description", "requirements", "benefits"):
|
||||
assert strip_html(data[key])[:40] in text
|
||||
assert text.index(strip_html(data["description"])[:40]) < \
|
||||
text.index(strip_html(data["benefits"])[:40])
|
||||
assert html_text(data[key]) in text
|
||||
assert text.index(html_text(data["description"])) < text.index(html_text(data["benefits"]))
|
||||
assert "<" not in text and " " not in text and len(text) > 1500
|
||||
|
||||
|
||||
@@ -96,7 +95,7 @@ def test_detail_joins_what_is_there_and_unescapes() -> None:
|
||||
body = json.dumps({"description": "<p>Build & run</p>", "requirements": None,
|
||||
"benefits": "<ul><li>Pension</li></ul>"}).encode()
|
||||
assert workable.detail(lambda url, **_: body, "acme", {"shortcode": "AAA111"}) == \
|
||||
{"description": "Build & run Pension"}
|
||||
{"description": "Build & run\n## Benefits\n\u2022 Pension"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"{}", b"[]", b'{"title": "x"}'])
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""`html_text`: a host's description HTML as lines that keep its headings,
|
||||
bullets and paragraphs, and `excerpt` leaving those marks out of a row."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.models import excerpt # noqa: E402
|
||||
from plugins.readers._shared import html_text # noqa: E402
|
||||
|
||||
|
||||
def test_blocks_become_lines_lists_bullets_and_headings_marks() -> None:
|
||||
markup = ("<h2>About</h2><p>We are <b>great</b>.</p>"
|
||||
"<p><strong>What you will do:</strong></p>"
|
||||
"<ul><li><p>Build & ship</p></li><li>Lead<ul><li>nested</li></ul></li></ul>"
|
||||
"<p>Line one<br>Line two</p><script>x = 1</script>")
|
||||
assert html_text(markup) == ("## About\nWe are great.\n## What you will do:\n"
|
||||
"• Build & ship\n• Lead\n• nested\n"
|
||||
"Line one\nLine two")
|
||||
|
||||
|
||||
def test_a_long_bold_paragraph_is_not_a_heading() -> None:
|
||||
loud = "<p><b>" + "Apply now " * 10 + "</b></p>"
|
||||
assert not html_text(loud).startswith("## ")
|
||||
|
||||
|
||||
def test_markup_escaped_once_more_is_read_as_markup() -> None:
|
||||
assert html_text("<p>Hello &amp; bye</p><ul><li>a</li></ul>") \
|
||||
== "Hello & bye\n• a"
|
||||
|
||||
|
||||
def test_plain_text_keeps_its_lines_and_typed_bullets_become_bullets() -> None:
|
||||
assert html_text("plain text\n\n* one\n- two\nR&D < 5") == \
|
||||
"plain text\n• one\n• two\nR&D < 5"
|
||||
|
||||
|
||||
def test_nothing_gives_nothing() -> None:
|
||||
assert html_text(None) == html_text("") == html_text("<p> </p>") == ""
|
||||
|
||||
|
||||
def test_the_row_excerpt_leaves_the_marks_out() -> None:
|
||||
assert excerpt("## About the role\n• Build & run care.") == "About the role Build & run care."
|
||||
Reference in new issue
Block a user