Runs is the queue's elements and a Problems log; a Kill lands between boards
Run one stage goes: the cascade is the unit of work and app job keeps the per-stage entry points. The pull button is disabled while a pull runs and the lede no longer hardcodes the cadence. Messages folds into Runs as Problems. pull_batch checks for a cancel before every board, one plugin per target, and both plugins now close their HTTP client. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
cebe62ddc9
commit
8532e4e739
15 files changed
+284
-242
No files matched your search
+45
-45
@@ -532,50 +532,50 @@ class ATSSource(Source):
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
with httpx.Client() as client:
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
continue
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
Reference in new issue
Block a user