Summarise and fetch text: a summary stage after score, two role routes
app/jobs/summarise.py is a ModelTask beside extract: about 60 words per role
from title, company, place and the fetched page else the description (capped
at 8000 characters). The pull cascade gains a summarise stage after score that
takes only roles without a summary in the top summarise.per_agent of an
enabled agent's ungated scores, and finds nothing when summarise.enabled is
false or no model is reachable. A reply that leaves a role unanswered is
counted in the state table and the role is given up after three.
POST /api/roles/{uid}/text scrapes original_url once and stores
article_text; POST /api/roles/{uid}/summarise queues a summarise_batch for
that uid alone, deduped per role.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
7d31cca394
commit
9035529e0d
12 files changed
+733
-10
No files matched your search
@@ -16,6 +16,10 @@ from fastapi import APIRouter, HTTPException, Query, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app import jobs
|
||||
from app.models import Role
|
||||
from desk.tools import agent, scrape
|
||||
from pipeline import Job
|
||||
from pipeline.web import QueueDep
|
||||
from app.models import (
|
||||
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights,
|
||||
)
|
||||
@@ -54,6 +58,73 @@ def list_roles(
|
||||
agent=agent, gated=gated)
|
||||
|
||||
|
||||
# ------------------------------------------------- roles: text and summary
|
||||
|
||||
#: What the read page toasts when a posting page gave nothing readable, by
|
||||
#: `desk.tools.failures` category.
|
||||
_UNREADABLE = {
|
||||
"timeout": "The posting page took too long to answer. Try again in a minute.",
|
||||
"unreachable": "The posting page could not be reached.",
|
||||
"rate_limited": "The posting site is refusing requests for now. Try again later.",
|
||||
"server_error": "The posting site answered with an error. Try again later.",
|
||||
"not_found": "The posting page is gone, so the role has probably closed.",
|
||||
"denied": "The posting site refused to show the page.",
|
||||
"consent_wall": "The posting page showed a consent banner instead of the posting.",
|
||||
"js_required": "The posting page only renders in a browser. Open it with the link instead.",
|
||||
"empty": "The posting page had no readable text on it.",
|
||||
}
|
||||
|
||||
|
||||
def _known_role(store: Store, uid: str) -> Role:
|
||||
role = store.role(uid)
|
||||
if role is None:
|
||||
raise HTTPException(404, f"no such role: {uid}")
|
||||
return role
|
||||
|
||||
|
||||
@router.post("/roles/{uid}/text")
|
||||
def fetch_text(uid: str, store: StoreDep) -> Role:
|
||||
"""Fetch the posting page once and keep it as plain text, for a role whose
|
||||
description is a teaser. Synchronous: one request the person asked for,
|
||||
and on demand rather than in the pull because each fetch of a Jobindex
|
||||
redirect counts as a click in its statistics."""
|
||||
role = _known_role(store, uid)
|
||||
if role.article_text:
|
||||
raise HTTPException(409, "The posting text is already fetched.")
|
||||
if not role.original_url:
|
||||
raise HTTPException(422, "This role has no link to fetch the posting from.")
|
||||
got = scrape.scrape(role.original_url)
|
||||
text = got.content.plain_text.strip() if got.content is not None else ""
|
||||
if not text:
|
||||
raise HTTPException(422, _UNREADABLE.get(
|
||||
got.failure, "Nothing readable came back from the posting page."))
|
||||
updated = store.set_article_text(uid, text)
|
||||
if updated is None:
|
||||
raise HTTPException(404, f"no such role: {uid}")
|
||||
return updated
|
||||
|
||||
|
||||
@router.post("/roles/{uid}/summarise", status_code=202)
|
||||
def summarise_role(uid: str, store: StoreDep, config: ConfigDep, queue: QueueDep) -> Job:
|
||||
"""Queue a summary of one role, whatever its rank and whether or not the
|
||||
pull summarises (`summarise.enabled`). The job's batch is this uid alone.
|
||||
Pressed twice, the second press gets the first one's job."""
|
||||
role = _known_role(store, uid)
|
||||
if role.summary:
|
||||
raise HTTPException(409, "This role already has a summary.")
|
||||
if not (role.article_text.strip() or role.description.strip()):
|
||||
raise HTTPException(422, "This role has no text to summarise. Fetch the posting first.")
|
||||
if not agent.backend(config.agent):
|
||||
raise HTTPException(422, "No model is reachable from here, so nothing can "
|
||||
"write a summary. Set up the API key or Claude Code.")
|
||||
job = queue.enqueue(f"{jobs.summarise.TASK.kind}_batch", {"uids": [uid]},
|
||||
label=f"Summarising {role.title or uid}",
|
||||
dedup_key=f"summarise:{uid}")
|
||||
if job is None:
|
||||
raise HTTPException(409, "The summary would not start.")
|
||||
return job
|
||||
|
||||
|
||||
# --------------------------------------------------------------- companies
|
||||
|
||||
class CompanyOut(BaseModel):
|
||||
|
||||
Reference in new issue
Block a user