Fetch a role's posting text through fetch_posting, so a Jobindex role is read from its Se jobbet link

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:47:06 +02:00
1 parent 8c66aeb445
commit 699bf81e3b
1 file changed
+7 -33
+7 -33
View File
@@ -21,7 +21,7 @@ from app.models import (
) )
from app.storage import Store from app.storage import Store
from app.web.deps import ConfigDep, StoreDep from app.web.deps import ConfigDep, StoreDep
from desk.tools import agent, scrape from desk.tools import agent
from pipeline import Job from pipeline import Job
from pipeline.web import QueueDep from pipeline.web import QueueDep
@@ -351,21 +351,6 @@ def place_labels(tokens: list[str] = Query(default=[])) -> list[PlaceLabel]:
# ------------------------------------------------- roles: text and summary # ------------------------------------------------- roles: text and summary
#: What the read page toasts when a posting page gave nothing readable, by
#: `desk.tools.failures` category.
_UNREADABLE = {
"timeout": "The posting page took too long to answer. Try again in a minute.",
"unreachable": "The posting page could not be reached.",
"rate_limited": "The posting site is refusing requests for now. Try again later.",
"server_error": "The posting site answered with an error. Try again later.",
"not_found": "The posting page is gone, so the role has probably closed.",
"denied": "The posting site refused to show the page.",
"consent_wall": "The posting page showed a consent banner instead of the posting.",
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
"empty": "The posting page had no readable text on it.",
}
def _known_role(store: Store, uid: str) -> Role: def _known_role(store: Store, uid: str) -> Role:
role = store.role(uid) role = store.role(uid)
if role is None: if role is None:
@@ -376,23 +361,12 @@ def _known_role(store: Store, uid: str) -> Role:
@router.post("/roles/{uid}/text") @router.post("/roles/{uid}/text")
def fetch_text(uid: str, store: StoreDep) -> Role: def fetch_text(uid: str, store: StoreDep) -> Role:
"""Fetch the posting page once and keep it as plain text, for a role whose """Fetch the posting page once and keep it as plain text, for a role whose
description is a teaser. Synchronous: one request the person asked for, description is a teaser (`jobs.posting.fetch_posting`); a Jobindex role
and on demand rather than in the pull because each fetch of a Jobindex is read from the page its "Se jobbet" link leads to."""
redirect counts as a click in its statistics.""" try:
role = _known_role(store, uid) return jobs.posting.fetch_posting(store, uid)
if role.article_text: except jobs.posting.PostingError as exc:
raise HTTPException(409, "The posting text is already fetched.") raise HTTPException(exc.status, str(exc)) from exc
if not role.original_url:
raise HTTPException(422, "This role has no link to fetch the posting from.")
got = scrape.scrape(role.original_url)
text = got.content.plain_text.strip() if got.content is not None else ""
if not text:
raise HTTPException(422, _UNREADABLE.get(
got.failure, "Nothing readable came back from the posting page."))
updated = store.set_article_text(uid, text)
if updated is None:
raise HTTPException(404, f"no such role: {uid}")
return updated
@router.post("/roles/{uid}/summarise", status_code=202) @router.post("/roles/{uid}/summarise", status_code=202)