diff --git a/README.md b/README.md index c46eb74..369d93c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic | | | |---|---| | **Search agents** | a card per agent, newest first: name, an On/Off switch, what it looks for, and its weights. Edit turns the card into the form in place (name, intent, weights, require/prefer gates behind an Advanced disclosure); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it | -| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Pull now, what is running, the schedule, one stage at a time, and the message log) | +| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Run the pull now, what is running, the schedule, and the Problems log) | | **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else | Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed. diff --git a/app/jobs/cascade.py b/app/jobs/cascade.py index f74beea..13ecc61 100644 --- a/app/jobs/cascade.py +++ b/app/jobs/cascade.py @@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str return {"started": " > ".join(CASCADES["pull"])} -#: What `app job ` and the Pulls page offer besides `pull`: one stage +#: What `app job ` and `/api/jobs` start besides `pull`: one stage #: each, on its own. `pull` is not a task here, since it is a job: a name that -#: is both would be two buttons on the Pulls page. +#: is both would be listed twice by `/api/jobs/kinds`. for _name in ("extract", "embed", "score", "summarise"): TASKS[_name] = partial(task, _name) @@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"): # ---------------------------------------------------------------- schedules # The default, next to the job it fires. A starting point rather than a -# setting: a cadence edited or switched off on the Pulls page survives +# setting: a cadence edited or switched off under Runs survives # every restart (see pipeline/schedule.py). declare("pull", "pull", "every 6h", label="Pull every board, then embed, score, extract, rescore and summarise") diff --git a/app/jobs/pull.py b/app/jobs/pull.py index e502d91..74e5cca 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = they list. `targets` is keyword-only without a default, which is what makes it a - required payload key to the queue: the Pulls page offers no button for a - batch that could only fail without one. + required payload key to the queue: `/api/jobs/kinds` lists no batch that + could only fail without one. - Builds the plugin scoped to just these targets — `Source.targets()` reads - `options[subscription_field]`, so handing it only this batch's entries is - what keeps one job from re-fetching every other job's boards. + One plugin per target, scoped to just that entry — `Source.targets()` + reads `options[subscription_field]` — so one job never re-fetches another + job's boards, and `context.check()` runs before each fetch: a Kill lands + between two boards rather than after the tenth. """ from plugins.ats import ATSSource from plugins.search import SearchSource # Fetchable now, not when the pull was queued: a board switched off on the - # Searches or Companies page since then is skipped. + # Job boards or Company boards since then is skipped. subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True) if s.target in targets} - options = dict(config.sources.get(source) or {}) - plugin: ATSSource | SearchSource - if source == "search": - options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params} - for t in targets if t in subs] - plugin = SearchSource(options) - else: - options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label} - for t in targets if t in subs] - plugin = ATSSource(options) - plugin.state = store.state - stored = 0 - for item in plugin.fetch(): - item.source = source + for target in targets: + sub = subs.get(target) + if sub is None: + continue + context.check() + options = dict(config.sources.get(source) or {}) + plugin: ATSSource | SearchSource if source == "search": - target = str(item.raw.get("search", "")) + options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] + plugin = SearchSource(options) else: - target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}" - store.upsert_role(item, subs.get(target)) - stored += 1 + options["boards"] = [{"url": target, "label": sub.company or sub.label}] + plugin = ATSSource(options) + plugin.state = store.state + for item in plugin.fetch(): + item.source = source + store.upsert_role(item, sub) + stored += 1 return {"roles": stored} diff --git a/app/jobs/summarise.py b/app/jobs/summarise.py index f0f30db..6090f6e 100644 --- a/app/jobs/summarise.py +++ b/app/jobs/summarise.py @@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL, def summarise_task() -> Task: - """Finds nothing while `summarise.enabled` is false, so the task, the - button on the Pulls page and the cascade stage are all no-ops with the - switch off.""" + """Finds nothing while `summarise.enabled` is false, so the task, `app + job summarise` and the cascade stage are all no-ops with the switch + off.""" return replace(modeltask.of(TASK), label=TASK.label) diff --git a/app/static/app.css b/app/static/app.css index 05d02f2..dbd1aaf 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -275,13 +275,6 @@ button.secondary { font-size: 0.8rem; } -/* -- pulls ------------------------------------------------------------------ */ -/* "Run one stage" on the Pulls page: extract, embed and score on their own, - under the schedule rather than beside Pull now. */ - -details.stage summary { cursor: pointer; color: var(--dim); font-size: 0.95rem; font-weight: 600; } -details.stage[open] summary { margin-bottom: 0.75rem; } - /* -- the notice log ------------------------------------------------------- */ .notice { diff --git a/app/static/jobs.js b/app/static/jobs.js index dd644ea..eb7852b 100644 --- a/app/static/jobs.js +++ b/app/static/jobs.js @@ -1,11 +1,12 @@ -/* The Runs section of Sources: Pull now on top, one button per remaining - * kind under "Run one stage". +/* The Runs section of Sources: "Run the pull now" and the idle line. * - * The list comes from `/api/jobs/kinds`, which is the registry, so a new - * `@job` in `app/jobs.py` is a new button and this file is not edited -- - * except that `pull` itself is pulled out of the list and wired to the - * dedicated button instead of appearing twice. What the queue is doing is - * ``'s business, and it draws itself. + * What the queue is doing and when it runs next are ``' and + * ``' business, and they draw themselves. The single stages + * have no buttons: the cascade is the unit of work, and `app job ` + * keeps the per-stage entry points. + * + * Both read the `app:jobs` snapshot from live.js rather than asking the + * elements, which keep to a shadow root and owe nobody an answer. * * Failures are the server's sentence through `api()`, which toasts it -- so * there is no error handling here beyond keeping the button usable. @@ -14,63 +15,57 @@ (function () { "use strict"; - const host = document.getElementById("kinds"); const pullButton = document.getElementById("pull-now"); - async function start(kind, button) { - button.disabled = true; - try { - const job = await window.api("/api/jobs", { method: "POST", body: { kind } }); - window.toast.good(`${job.label || kind}: queued.`); - } catch (e) { - /* already toasted */ - } finally { - button.disabled = false; - } + let pending = false; // the POST is in the air + let pulling = false; // the latest snapshot has a live pull cascade + + /* A pull queued by the button or a schedule is the `pull` job, which + begins a cascade of its own named `pull on its own`, and the chain's + tasks join it; `pull` is the name a cascade started as one would have. + Live while any task has a job queued or running: the pipeline keeps a + finished cascade in the snapshot for a few seconds, and that is not a + pull still going. As newsdesk's Fetch: the whole cascade, not the `pull` + task, which is over seconds in with five tasks to go. */ + const isPull = (c) => c.name === "pull" || c.name === "pull on its own"; + const live = (c) => (c.tasks || []).some((t) => t.running || t.queued); + + function paint() { + if (!pullButton) return; + pullButton.disabled = pending || pulling; + pullButton.title = pulling ? "A pull is running." : ""; } if (pullButton) { - pullButton.addEventListener("click", () => start("pull", pullButton)); - } - - async function draw() { - if (!host) return; - let kinds; - try { - kinds = await window.api("/api/jobs/kinds"); - } catch (e) { - host.innerHTML = 'could not read the job kinds'; - return; - } - kinds = kinds.filter((k) => k.kind !== "pull"); - if (!kinds.length) { - host.innerHTML = 'nothing is registered'; - return; - } - host.innerHTML = ""; - kinds.forEach((k) => { - const button = document.createElement("button"); - button.type = "button"; - button.className = "secondary"; - button.textContent = k.label || k.kind; - button.title = k.solo ? `${k.kind} -- one at a time` : k.kind; - button.dataset.kind = k.kind; - button.addEventListener("click", () => start(k.kind, button)); - host.appendChild(button); + pullButton.addEventListener("click", async () => { + pending = true; + paint(); + try { + const job = await window.api("/api/jobs", { method: "POST", body: { kind: "pull" } }); + window.toast.good(`${job.label || "pull"}: queued.`); + } catch (e) { + /* already toasted */ + } finally { + pending = false; + paint(); + } }); } /* `` takes up no space while the queue is idle, so the line that - says so is the app's to show and hide. Off the same socket the element - reads, through live.js, rather than by asking the element: it keeps to a - shadow root and owes nobody an answer about its own height. */ - document.addEventListener("app:jobs", (e) => { + says so is the app's to show and hide. It starts hidden: until the first + snapshot nobody knows whether the queue is idle. */ + function update(state) { const idle = document.getElementById("idle"); - if (!idle) return; - const state = e.detail || {}; - const busy = (state.running || []).length + (state.queued || []).length; - idle.hidden = busy > 0; - }); + if (idle) { + const busy = (state.running || []).length + (state.queued || []).length; + idle.hidden = busy > 0; + } + pulling = (state.cascades || []).some((c) => isPull(c) && live(c)); + paint(); + } - document.addEventListener("DOMContentLoaded", draw); + document.addEventListener("app:jobs", (e) => update(e.detail || {})); + // A snapshot that arrived before this script loaded. + if (window.live && window.live.jobs) update(window.live.jobs); })(); diff --git a/app/static/sources.js b/app/static/sources.js index cd3d319..5f18cf9 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -206,17 +206,19 @@ // ------------------------------------------------------ one section shown - // Sources shows one section: the one the fragment names, or the first when - // there is none or it names none, so `/settings/sources#runs` (a rail link, - // an old path's 301, the Search agents page's "follow it under Runs") opens - // Runs. The rest are hidden, not removed, so their scripts still find their - // roots. It runs as the script loads rather than on DOMContentLoaded, so the - // page never paints all four first. + // Sources shows one section: the one the fragment names or contains, or the + // first when there is none or it names none, so `/settings/sources#runs` (a + // rail link, an old path's 301, the Search agents page's "follow it under + // Runs") opens Runs, and so does `#messages`, the log at its end. The rest + // are hidden, not removed, so their scripts still find their roots. It runs + // as the script loads rather than on DOMContentLoaded, so the page never + // paints all three first. function showSection() { const sections = [...document.querySelectorAll("section[data-section]")]; if (!sections.length) return; const want = decodeURIComponent(location.hash.slice(1)); - const shown = sections.find((s) => s.id === want) || sections[0]; + const target = want && document.getElementById(want); + const shown = sections.find((s) => target && s.contains(target)) || sections[0]; for (const section of sections) section.hidden = section !== shown; for (const link of document.querySelectorAll(".settings-rail a[data-section]")) { if (link.dataset.section === shown.id) { @@ -229,10 +231,13 @@ } showSection(); - // A switch starts the section at the top of the page, as opening it would. + // A switch starts the section at the top of the page, as opening it would; + // a fragment inside one (`#messages`) scrolls to itself. window.addEventListener("hashchange", () => { showSection(); - window.scrollTo(0, 0); + const target = document.getElementById(decodeURIComponent(location.hash.slice(1))); + if (target && !target.matches("section[data-section]")) target.scrollIntoView(); + else window.scrollTo(0, 0); }); // -------------------------------------------------------------------- init diff --git a/app/templates/settings/account.html b/app/templates/settings/account.html index 939142d..dfce2e1 100644 --- a/app/templates/settings/account.html +++ b/app/templates/settings/account.html @@ -6,7 +6,7 @@

The one password, the devices signed in with it, and the passkeys that sign in without it.

- {# As on the Pulls page: the element and its script belong to + {# As in Sources' Runs: the element and its script belong to the module, so this page is two lines and `auth/` owns what they draw. It says its own failures in place, beside the button that caused them, diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html index a837115..90f2366 100644 --- a/app/templates/settings/sources.html +++ b/app/templates/settings/sources.html @@ -1,14 +1,15 @@ {% extends "settings/_layout.html" %} {% block title %}Sources — roledesk{% endblock %} -{# One page, four sections: the `search` plugin's job board queries, the - `ats` plugin's company boards, the runs that read both, and the message - log. One is shown at a time, the one the URL fragment names (`#runs`), or - Job boards without one; sources.js hides the rest and the rail picks one. - Hidden rather than separate pages, because each section keeps the element - ids it had as a page of its own and its script (sources.js, companies.js, - jobs.js, messages.js) is unchanged: it finds its root or does nothing. The - ids are `SECTIONS` in app/web/pages.py, which the rail links to. #} +{# One page, three sections: the `search` plugin's job board queries, the + `ats` plugin's company boards, and the runs that read both, with the + message log as their last block. One is shown at a time, the one the URL + fragment names (`#runs`) or contains (`#messages`), or Job boards without + one; sources.js hides the rest and the rail picks one. Hidden rather than + separate pages, because each section keeps the element ids it had as a + page of its own and its script (sources.js, companies.js, jobs.js, + messages.js) is unchanged: it finds its root or does nothing. The ids are + `SECTIONS` in app/web/pages.py, which the rail links to. #} {% block content %}

Sources

Sources are where roles come from: queries on job boards that span companies, and the boards of the companies you follow. Each section is one plugin; the runs read them all.

@@ -109,18 +110,18 @@

Runs

- +
-

Every 6 hours the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries. Change the cadence below.

+

On the schedule below the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries.

What is running

{# The heading and the idle line are the app's: the element takes up no space while the queue is idle — that is its `idle="hide"` default, and a strip that is always there is furniture — and a heading with nothing - under it reads as something that failed to load. The element and its - script are the queue's own, served by its own router — see - pipeline/web/static/jobs.js. #} -

Nothing at the moment.

+ under it reads as something that failed to load. The line is hidden + until jobs.js has the first snapshot. The element and its script are the + queue's own, served by its own router — see pipeline/web/static/jobs.js. #} + @@ -132,22 +133,16 @@ -
- Run one stage -
- loading… + {# The message log, the queue's own through `/api/notices`. `#messages` is + where the old Messages paths and the toasts' "all messages" link land. #} +
+
+

Problems

+
-
- -
- -
-
-

Messages

- +

What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.

+
loading…
-

What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.

-
loading…
{% endblock %} diff --git a/app/web/pages.py b/app/web/pages.py index c4ba9b2..8106fe8 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -16,17 +16,19 @@ one page, so a link to a role and the role opened from the feed are the same screen. What the server renders is only what a select needs to offer, so the options cannot drift from what the store accepts. -Sources is one page in four sections: job boards (the `search` plugin's -queries), company boards (`ats`), the runs that read them, and the message -log. One is shown at a time, the one the URL fragment names; the others stay -in the page, hidden, so each section's script (the ones they had as pages of -their own, each guarding on its own root element) keeps working. +Sources is one page in three sections: job boards (the `search` plugin's +queries), company boards (`ats`), and the runs that read them, which end in +the message log. One is shown at a time, the one the URL fragment names or +contains; the others stay in the page, hidden, so each section's script (the +ones they had as pages of their own, each guarding on its own root element) +keeps working. Every path a page has had before (`/agents`, `/companies`, `/sources`, `/jobs`, `/schedules`, `/messages`, `/account`, and the six sub-tabs the settings shell started with) answers 301 to its place now, query string kept, so a bookmark still lands. A merged page's old path lands on its -section's anchor. +section's anchor; the old Messages paths land on `#messages`, the log's +block at the end of Runs. `FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the classifier's vocabulary, and this reaches `plugins/` the same way `app.cli` @@ -88,12 +90,12 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = { ("job-boards", "Job boards"), ("company-boards", "Company boards"), ("runs", "Runs"), - ("messages", "Messages"), ), } #: Where each page that used to be a tab or a sub-tab lives now. The fragment -#: is the section on the merged page; a browser keeps it across the 301. +#: is the section on the merged page, or an element in one (`#messages`, in +#: Runs); a browser keeps it across the 301. MOVED: dict[str, str] = { "/agents": "/settings/agents", "/settings/ranking": "/settings/agents", diff --git a/docs/decisions.md b/docs/decisions.md index 80c79b9..4ee0dc5 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -112,6 +112,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Settings pages sit beside a rail, and Sources shows one section at a time, chosen by the URL fragment.** The company boards list runs to 50 or 60 companies, so Runs at the end of a stacked page was out of reach and jump links still meant scrolling back up. The rail (`settings/_rail.html`, from `SETTINGS` and `SECTIONS` in `app/web/pages.py`) is sticky beside the content, and a strip under the header below 60rem. The fragment rather than a path per section, because the old paths already 301 to `/settings/sources#runs` and the like, and a hidden section stays in the DOM, so each section's script runs unchanged. +**Runs is the queue's own elements plus a Problems log.** `` and `` draw what is running and when the pull runs next, the app adds only "Run the pull now", disabled while a pull cascade is live, and the message log is the section's last block rather than a section of its own; `#messages` still lands on it. "Run one stage" went because the cascade is the unit of work: a stage run alone is a partial pull whose result the next stage never sees. `app job ` and `POST /api/jobs` keep the per-stage entry points, which is what "Save and score" uses. + **Search agents are cards, edited in place.** An agent is defined by what it looks for, so the intent is the card's body and the weights a small line under it; a list of names and weights hid the part that matters. Edit turns that card into the form and New puts a blank one on top, because the new agent sorts first by id and a form at the bottom of the page was out of sight of the card it changed. **Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working. diff --git a/plugins/ats.py b/plugins/ats.py index f4d4959..add18f3 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -532,50 +532,50 @@ class ATSSource(Source): ) def fetch(self) -> Iterator[Item]: - client = httpx.Client() - # Written by the pull job before any board is fetched (see - # `app/jobs/pull.py`); a plugin run on its own — a test, a dry - # run — has no run to stamp itself against, so it makes one up. - run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat( - timespec="seconds") + with httpx.Client() as client: + # Written by the pull job before any board is fetched (see + # `app/jobs/pull.py`); a plugin run on its own — a test, a dry + # run — has no run to stamp itself against, so it makes one up. + run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat( + timespec="seconds") - for target, label in self.targets(): - host, _, slug = target.partition(":") - try: - rows = _fetch_rows(client, target) - except Exception as exc: - log.warning("ats: %s failed (%s)", target, exc) - continue + for target, label in self.targets(): + host, _, slug = target.partition(":") + try: + rows = _fetch_rows(client, target) + except Exception as exc: + log.warning("ats: %s failed (%s)", target, exc) + continue - seen_ids = [] - for row in rows: - # The compound id, matching `roles.source_id` — what - # `Store.mark_closed` compares against, not the board's own - # bare job id. - compound_id = f"{host}:{slug}:{row['source_id']}" - seen_ids.append(compound_id) - yield Item( - source_id=compound_id, - location=row["url"], - title=row["title"], - excerpt=(row["description"] or "")[:300], - body=row["description"], - published_at=row["posted"], - subscription=label, - kind="role", - raw={ - "company": label, - "place": row["location"], - "department": row["department"], - "function": classify(row["department"], row["title"]), - "remote": row["remote"], - "salary": row["salary"], - "board": host, - "slug": slug, - }, - ) - # Stamped only once the board has parsed without raising, so a - # board that blew up half way through is neither marked ok nor - # has a partial id list treated as the whole of it. - self.state[f"ats:ok:{target}"] = run_marker - self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) + seen_ids = [] + for row in rows: + # The compound id, matching `roles.source_id` — what + # `Store.mark_closed` compares against, not the board's own + # bare job id. + compound_id = f"{host}:{slug}:{row['source_id']}" + seen_ids.append(compound_id) + yield Item( + source_id=compound_id, + location=row["url"], + title=row["title"], + excerpt=(row["description"] or "")[:300], + body=row["description"], + published_at=row["posted"], + subscription=label, + kind="role", + raw={ + "company": label, + "place": row["location"], + "department": row["department"], + "function": classify(row["department"], row["title"]), + "remote": row["remote"], + "salary": row["salary"], + "board": host, + "slug": slug, + }, + ) + # Stamped only once the board has parsed without raising, so a + # board that blew up half way through is neither marked ok nor + # has a partial id list treated as the whole of it. + self.state[f"ats:ok:{target}"] = run_marker + self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) diff --git a/plugins/search.py b/plugins/search.py index 28fe253..c73057d 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -308,46 +308,46 @@ class SearchSource(Source): ) def fetch(self) -> Iterator[Item]: - client = httpx.Client() - run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( - timespec="seconds") - for target, label in self.targets(): - provider = provider_of(target) - if not provider: - log.warning("search: %r is not provider:name; skipped", target) - continue - try: - rows, complete = _SEARCHES[provider]( - client, self.params_for(target), self.options.get(provider) or {}) - except Exception as exc: - log.warning("search: %s failed (%s)", target, exc) - continue + with httpx.Client() as client: + run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( + timespec="seconds") + for target, label in self.targets(): + provider = provider_of(target) + if not provider: + log.warning("search: %r is not provider:name; skipped", target) + continue + try: + rows, complete = _SEARCHES[provider]( + client, self.params_for(target), self.options.get(provider) or {}) + except Exception as exc: + log.warning("search: %s failed (%s)", target, exc) + continue - seen_ids = [] - for row in rows: - source_id = f"{provider}:{row['id']}" - seen_ids.append(source_id) - yield Item( - source_id=source_id, - location=row["url"], - title=row["title"], - excerpt=(row["description"] or "")[:300], - body=row["description"], - published_at=row["posted"], - subscription=label, - kind="role", - raw={ - "company": row["company"], - "place": row["place"], - "country": row["country"], - "remote": row["remote"], - "salary_min": row["salary_min"], - "salary_max": row["salary_max"], - "currency": row["currency"], - "provider": provider, - "search": target, - }, - ) - self.state[f"search:ok:{target}"] = run_marker - self.state[f"search:complete:{target}"] = "1" if complete else "" - self.state[f"search:ids:{target}"] = json.dumps(seen_ids) + seen_ids = [] + for row in rows: + source_id = f"{provider}:{row['id']}" + seen_ids.append(source_id) + yield Item( + source_id=source_id, + location=row["url"], + title=row["title"], + excerpt=(row["description"] or "")[:300], + body=row["description"], + published_at=row["posted"], + subscription=label, + kind="role", + raw={ + "company": row["company"], + "place": row["place"], + "country": row["country"], + "remote": row["remote"], + "salary_min": row["salary_min"], + "salary_max": row["salary_max"], + "currency": row["currency"], + "provider": provider, + "search": target, + }, + ) + self.state[f"search:ok:{target}"] = run_marker + self.state[f"search:complete:{target}"] = "1" if complete else "" + self.state[f"search:ids:{target}"] = json.dumps(seen_ids) diff --git a/tests/test_pull.py b/tests/test_pull.py index 6e3176e..191bfec 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -16,6 +16,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from desk.models import Item # noqa: E402 +from pipeline import Cancelled, RunContext # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app.jobs import pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -81,6 +82,35 @@ def test_run_batch_upserts_every_item_the_plugin_yields( assert store.roles()[0].subscription_id is not None +def test_a_kill_stops_the_batch_before_the_next_board( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """`check()` before each board: a Kill pressed while the first is read + ends the job before the second is fetched, and keeps what the first + stored.""" + for slug in ("acme", "beta"): + store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(), + company=slug.title(), board="greenhouse") + fetched: list[str] = [] + killed = False + + def fake_fetch(self: ATSSource) -> object: + nonlocal killed + [(target, _)] = self.targets() + fetched.append(target) + yield _item("greenhouse", target.partition(":")[2], "1") + killed = True + + monkeypatch.setattr(ATSSource, "fetch", fake_fetch) + context = RunContext(is_cancelled=lambda: killed) + + with pytest.raises(Cancelled): + pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"], + context=context) + + assert fetched == ["greenhouse:acme"] + assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} + + def test_close_boards_closes_what_a_completed_board_no_longer_lists( store: Store) -> None: sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", diff --git a/tests/test_web.py b/tests/test_web.py index 3734c99..bcb0515 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -549,18 +549,22 @@ def test_every_settings_page_has_the_rail_and_not_the_tab_row(client: TestClient def test_a_moved_section_is_one_the_sources_page_shows(client: TestClient) -> None: """Every old path that lands on a fragment of Sources names a section of - `SECTIONS`, and each is a `
` on the page, which is - what sources.js shows one of.""" + `SECTIONS` or an element inside one, and each section is a `
` on the page, which is what sources.js shows one of. The + old Messages paths land on `#messages`, the log at the end of Runs.""" anchors = [anchor for anchor, _ in SECTIONS["/settings/sources"]] - assert anchors == ["job-boards", "company-boards", "runs", "messages"] - for to in MOVED.values(): - path, _, anchor = to.partition("#") - if anchor: - assert path == "/settings/sources" and anchor in anchors, to + assert anchors == ["job-boards", "company-boards", "runs"] page = client.get("/settings/sources").text assert page.count("data-section>") == len(anchors) for anchor in anchors: assert f'
' in page + runs = page[page.index('id="runs"'):] + for to in MOVED.values(): + path, _, anchor = to.partition("#") + if anchor: + assert path == "/settings/sources", to + assert anchor in anchors or f'id="{anchor}"' in runs, to + assert MOVED["/messages"] == MOVED["/settings/messages"] == "/settings/sources#messages" def test_settings_is_search_agents(client: TestClient) -> None: @@ -586,8 +590,9 @@ def test_an_old_tab_moves_permanently_and_keeps_its_query( def test_each_page_loads_only_its_own_script(client: TestClient) -> None: - """Sources is four sections' scripts on one page, each guarding on its own - root element; every other page is one.""" + """Sources is four scripts on one page, one per section and one for the + log in Runs, each guarding on its own root element; every other page is + one.""" scripts = {"roles.js", "applications.js", "agents.js", "companies.js", "sources.js", "jobs.js", "messages.js"} for path, container, own in [ @@ -632,14 +637,15 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None: - """Job boards, company boards, the runs and the message log, in that - order, each linked from the rail; no jump row any more.""" + """Job boards, company boards and the runs, in that order, each linked + from the rail; no jump row any more.""" page = client.get("/settings/sources").text assert '
", page.index('id="runs"'))] + assert '' in runs + assert 'id="kinds"' not in page and "Run one stage" not in page + assert "On the schedule below the pull reads every board" in runs + assert "Every 6 hours" not in runs + assert '
') \ + < runs.index("

Problems

") < runs.index('id="clear-all"') < runs.index('id="notices"') def test_the_modules_bring_their_own_elements(client: TestClient) -> None: @@ -685,6 +698,14 @@ def test_the_jobs_tab_offers_pull_once_and_no_batch(client: TestClient) -> None: assert not {"pull_batch", "extract_batch", "advance"} & set(kinds) +def test_save_and_score_queues_a_score(client: TestClient, queue: Any) -> None: + """The Search agents page's "Save and score" posts `{kind: "score"}`: a + task on its own, now that no stage has a button of its own.""" + res = client.post("/api/jobs", json={"kind": "score", "payload": {}}) + assert res.status_code == 202 + assert queue.jobs.last("score", states=("queued", "running", "done")) is not None + + def test_an_error_is_one_sentence(client: TestClient) -> None: """`api()` toasts `detail` as it is, so a 422 is a sentence, not a list.""" res = client.post("/api/agents", json={"name": "Bad",