Merge branch 'worktree-agent-a3bd292551395dda0'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
43773efc53
8 files changed
+338
-34
No files matched your search
+8
-5
@@ -17,7 +17,7 @@ roles out of extract's reach for good.
|
||||
fit cos(role vector, intent vector), min-max scaled across
|
||||
the roles this agent scores in this run: the closest is 1,
|
||||
the farthest 0, a lone role 1
|
||||
recency 0.5 ** (days since first_seen / 14)
|
||||
recency 0.5 ** (days since posted_at / 14)
|
||||
affinity (the agent's affinity for the company + 1) / 3
|
||||
taste cos(role vector, liked direction), min-max scaled like fit,
|
||||
where the direction is mean(liked vectors) - mean(passed vectors)
|
||||
@@ -66,10 +66,13 @@ SEEDED = "agents:seeded"
|
||||
|
||||
# ------------------------------------------------------------------- terms
|
||||
|
||||
def recency(first_seen: str, now: datetime) -> float:
|
||||
"""1.0 for a role first seen now, halving every `HALF_LIFE_DAYS`."""
|
||||
def recency(posted_at: str, now: datetime) -> float:
|
||||
"""1.0 for a role posted now, halving every `HALF_LIFE_DAYS`. `posted_at`
|
||||
is `app.models.posted_at`: the earlier of when the app first saw the role
|
||||
and, for a trusted host, when the board says it was posted — so a
|
||||
months-old backlog a newly added board brings in does not start at 1.0."""
|
||||
try:
|
||||
seen = datetime.fromisoformat(first_seen)
|
||||
seen = datetime.fromisoformat(posted_at)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
days = max((now - seen).total_seconds(), 0.0) / 86400
|
||||
@@ -165,7 +168,7 @@ def score_one(candidate: Candidate, gates: Sequence[Gate], weights: Weights,
|
||||
if failed := failed_gates(candidate, gates):
|
||||
return 0.0, ", ".join(failed)
|
||||
|
||||
terms = [(weights.recency, recency(candidate.first_seen, now)),
|
||||
terms = [(weights.recency, recency(candidate.posted_at, now)),
|
||||
(weights.affinity, affinity_term(candidate.affinity))]
|
||||
terms += [(g.weight, 1.0 if state == 1 else 0.0)
|
||||
for g, state in zip(gates, candidate.gates, strict=True) if g.mode == "prefer"]
|
||||
|
||||
+81
-4
@@ -228,6 +228,79 @@ STATUSES: tuple[str, ...] = get_args(Status)
|
||||
AFFINITY = range(-1, 3)
|
||||
|
||||
|
||||
# -------------------------------------------------------------- posting dates
|
||||
|
||||
#: Hosts whose date is a creation or first-publication timestamp, safe to
|
||||
#: read as when a role was posted, checked against live boards on
|
||||
#: 2026-09-29:
|
||||
#:
|
||||
#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers
|
||||
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
||||
#: with `first_published` five months before `updated_at`, proving the
|
||||
#: first stays fixed while the second moves on an edit — the resettable
|
||||
#: field decisions.md used to warn against is `updated_at`, which this
|
||||
#: reads only when `first_published` is missing (rare: 0 of 208 on that
|
||||
#: board). The large real gaps this migration measured for Greenhouse
|
||||
#: roles (average 143 days between this date and `first_seen`) are further
|
||||
#: evidence it holds still rather than tracking every edit.
|
||||
#: - `lever` (`createdAt`), `ashby` (`publishedAt`), `personio` (`createdAt`):
|
||||
#: named and shaped as creation dates, with no "last edited" field in the
|
||||
#: feed at all to reset from, and the same large real gaps as Greenhouse's.
|
||||
#: - `smartrecruiters` (`releasedDate`), `recruitee` (`published_at`, falling
|
||||
#: back to `created_at`): `releasedDate` reads the same way; a live
|
||||
#: Recruitee board (bunq) showed `published_at` and `created_at` minutes
|
||||
#: apart while `updated_at` had moved days later on an edited posting,
|
||||
#: confirming the field this migration reads is not the one that moves.
|
||||
#: - `workable` (`published`): named as a publish date, not an edit one, but
|
||||
#: only one live board had postings to check (huggingface, 8 roles) and
|
||||
#: none showed a second, "last touched" field to compare against, so this
|
||||
#: is trusted on the name and shape alone.
|
||||
#: - `jobindex`: the RSS `pubDate` a saved search returns. The live
|
||||
#: database's 158 Jobindex-found roles have a `published_at`/`first_seen`
|
||||
#: gap of 0.4 to 31 days (median 12), which reads as real, spread-out
|
||||
#: posting ages rather than a date that jumps to "now" on every re-crawl.
|
||||
#: - `adzuna`: `created`, an aggregator's own record-creation stamp — named
|
||||
#: as creation, not modification, like the rest. No live check: this
|
||||
#: deployment carries no Adzuna key (`plugins.search`'s own docstring:
|
||||
#: "not yet run live") and no Adzuna subscription exists to fetch.
|
||||
#:
|
||||
#: A host not listed here, or a role whose stored `published_at` fails the
|
||||
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
|
||||
#: safe default this migration is replacing everywhere else.
|
||||
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
|
||||
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
||||
"personio", "jobindex", "adzuna",
|
||||
})
|
||||
|
||||
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
||||
#: real posting date: no board here existed before 2015.
|
||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def posted_host(raw: dict[str, Any]) -> str:
|
||||
"""The host a role's dates came from, read the way `ats` and `search`
|
||||
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
||||
`lever`, ...), `provider` for a cross-company search (`jobindex`,
|
||||
`adzuna`). '' when neither plugin wrote one."""
|
||||
return str(raw.get("board") or raw.get("provider") or "")
|
||||
|
||||
|
||||
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
||||
"""The date a role counts as posted: the earlier of `first_seen` (when
|
||||
the app itself first stored it) and `published_at` (what the board
|
||||
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
|
||||
`published_at` is neither absurdly old nor later than `first_seen` — a
|
||||
role cannot be posted after the app already saw it, trusted host or not,
|
||||
which also catches a `published_at` the board reports in the future
|
||||
(`first_seen` is never later than the moment this runs). Both dates are
|
||||
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
||||
is exact and needs no parsing. A role with no trusted date, or with one
|
||||
that fails the check, keeps `first_seen`."""
|
||||
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
|
||||
return published_at
|
||||
return first_seen
|
||||
|
||||
|
||||
def _json(value: Any, empty: str) -> Any:
|
||||
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
||||
value that is already parsed passes, and one that does not parse is the
|
||||
@@ -273,6 +346,10 @@ class Role(Model):
|
||||
feed_text: str = ""
|
||||
original_url: str = ""
|
||||
published_at: str
|
||||
#: The date this role counts as posted: see `posted_at`. Used for
|
||||
#: `recency` and the feed's newest order; `first_seen` stays what it
|
||||
#: always was, when the app itself first stored the role.
|
||||
posted_at: str
|
||||
first_seen: str
|
||||
last_seen: str
|
||||
closed_at: str | None = None
|
||||
@@ -448,12 +525,12 @@ class Agent(Model):
|
||||
|
||||
|
||||
class Candidate(Model):
|
||||
"""What scoring reads about one open role for one agent: when it was
|
||||
first seen, its company's affinity for that agent, and per gate `1`
|
||||
match, `0` no match, `-1` the column is empty."""
|
||||
"""What scoring reads about one open role for one agent: its posted
|
||||
date (for `recency`), its company's affinity for that agent, and per
|
||||
gate `1` match, `0` no match, `-1` the column is empty."""
|
||||
|
||||
uid: str
|
||||
first_seen: str
|
||||
posted_at: str
|
||||
affinity: int = 0
|
||||
gates: list[int] = []
|
||||
|
||||
|
||||
+1
-1
@@ -509,7 +509,7 @@
|
||||
${role.company ? `<b>${esc(role.company)}</b>` : ""}
|
||||
${role.place ? `<span>${esc(role.place)}</span>` : ""}
|
||||
${badges(role)}
|
||||
<span title="First seen">${ago(role.first_seen)}</span>
|
||||
<span title="Posted">${ago(role.posted_at)}</span>
|
||||
${role.gated_reason
|
||||
? `<span class="gated" title="Kept out of the ranking">${esc(role.gated_reason)}</span>`
|
||||
: ""}
|
||||
|
||||
+36
-3
@@ -12,10 +12,13 @@ shipped.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.models import posted_at, posted_host
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """
|
||||
@@ -56,9 +59,15 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
feed_text TEXT NOT NULL DEFAULT '', -- the teaser a feed handed over
|
||||
original_url TEXT NOT NULL DEFAULT '', -- the posting page
|
||||
published_at TEXT NOT NULL, -- what the source claims, UTC
|
||||
-- What storing postings buys. first_seen is the only honest posted date,
|
||||
-- since Greenhouse resets updated_at on every edit. closed_at is set when a
|
||||
-- board fetched in full no longer lists the role, and cleared if it returns.
|
||||
-- What storing postings buys. first_seen is when the app itself first
|
||||
-- stored the role; posted_at (app.models.posted_at) is the earlier of
|
||||
-- first_seen and published_at, trusted only for a host known to report a
|
||||
-- creation date rather than one that moves on an edit
|
||||
-- (app.models.TRUSTED_POSTED_HOSTS) — recency and the feed's newest order
|
||||
-- read posted_at, first_seen stays "when the app found it". closed_at is
|
||||
-- set when a board fetched in full no longer lists the role, and cleared
|
||||
-- if it returns.
|
||||
posted_at TEXT NOT NULL DEFAULT '',
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
closed_at TEXT,
|
||||
@@ -244,6 +253,11 @@ CREATE INDEX IF NOT EXISTS idx_roles_company ON roles(company_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_place ON roles(place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_first_seen ON roles(first_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_closed ON roles(closed_at);
|
||||
-- Serves the newest feed (open_only, ordered by posted_at DESC), which is
|
||||
-- most of what `Store.roles` runs: partial, since only an open role's
|
||||
-- position in that order is ever asked for.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_posted
|
||||
ON roles(posted_at DESC, uid) WHERE closed_at IS NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_cluster ON roles(company_key, title_key, place_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_board ON roles(source, subscription_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_application_events
|
||||
@@ -260,6 +274,23 @@ MARKER = "roles"
|
||||
|
||||
Step = str | Callable[[sqlite3.Connection], None]
|
||||
|
||||
|
||||
def _migrate_posted_at(conn: sqlite3.Connection) -> None:
|
||||
"""v10's step: add `posted_at`, then backfill it for every row already
|
||||
stored, by the same rule `Store.upsert_role` applies to a new one (see
|
||||
`app.models.posted_at`). A callable, not SQL, because the rule reads
|
||||
`raw` for which host a role came from. `'' NOT NULL` is a placeholder
|
||||
only until the UPDATE below runs, in the same transaction as the ALTER
|
||||
(`_apply` wraps every migration in one); every row gets a real value."""
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posted_at TEXT NOT NULL DEFAULT ''")
|
||||
rows = conn.execute("SELECT uid, published_at, first_seen, raw FROM roles").fetchall()
|
||||
conn.executemany(
|
||||
"UPDATE roles SET posted_at = ? WHERE uid = ?",
|
||||
[(posted_at(row["published_at"], row["first_seen"],
|
||||
posted_host(json.loads(row["raw"] or "{}"))), row["uid"])
|
||||
for row in rows])
|
||||
|
||||
|
||||
MIGRATIONS: tuple[tuple[int, str, Step], ...] = (
|
||||
# Never run as a migration: a file without the marker table is created
|
||||
# from `SCHEMA` and stamped. It exists so the first real migration is 2.
|
||||
@@ -344,6 +375,8 @@ ALTER TABLE agents ADD COLUMN limits TEXT NOT NULL DEFAULT '{}';
|
||||
(9, "companies.ats_slug: the slug of a host the app cannot read yet is kept", """
|
||||
ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(10, "roles.posted_at: recency and the newest feed count from when a role "
|
||||
"was posted, not just when the app found it", _migrate_posted_at),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
|
||||
+29
-10
@@ -23,7 +23,7 @@ from desk.models import Item
|
||||
from app.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
|
||||
country_of, place_key, remote_of, title_key,
|
||||
country_of, place_key, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.storage import schema
|
||||
|
||||
@@ -263,6 +263,12 @@ class Store:
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
|
||||
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
||||
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
||||
keeps the earlier of that and what is already stored, so an edit on
|
||||
the board — which can only ever move `published_at` later, never
|
||||
earlier — cannot move `posted_at` later either.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
@@ -270,6 +276,7 @@ class Store:
|
||||
company = str(raw.get("company") or (subscription.company if subscription else ""))
|
||||
place = str(raw.get("place") or "")
|
||||
stamp = self.now()
|
||||
published = _iso(item.published_at)
|
||||
row: dict[str, Any] = {
|
||||
"uid": item.uid,
|
||||
"source": item.source,
|
||||
@@ -284,7 +291,8 @@ class Store:
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
"published_at": _iso(item.published_at),
|
||||
"published_at": published,
|
||||
"posted_at": posted_at(published, stamp, posted_host(raw)),
|
||||
"first_seen": stamp,
|
||||
"last_seen": stamp,
|
||||
"raw": json.dumps(raw, default=str),
|
||||
@@ -302,6 +310,7 @@ class Store:
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -429,12 +438,12 @@ class Store:
|
||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||
judged: str | None = None, offset: int = 0) -> list[ScoredRole]:
|
||||
"""Newest first by `first_seen`. `company` and `place` are
|
||||
"""Newest first by `posted_at`. `company` and `place` are
|
||||
case-insensitive substrings of the columns a board wrote — not the
|
||||
key, so a search does not have to know the exact slug a company folds
|
||||
to. `q` is matched word by word against `title`; a role must match
|
||||
every word. `function` and `remote` match a gate column exactly.
|
||||
`days` keeps only roles first seen within that many days of now.
|
||||
`days` keeps only roles posted within that many days of now.
|
||||
|
||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
||||
@@ -471,10 +480,20 @@ class Store:
|
||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||
judged=judged)
|
||||
select = "SELECT roles.*, v.vote, v.voted_at"
|
||||
order = "roles.first_seen DESC, roles.uid"
|
||||
if agent is not None and by_score:
|
||||
order = "roles.posted_at DESC, roles.uid"
|
||||
scored = agent is not None and by_score
|
||||
if scored:
|
||||
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
|
||||
order = "score DESC, " + order
|
||||
elif open_only:
|
||||
# Newest first, open only: idx_roles_posted covers this exactly
|
||||
# (posted_at DESC, uid, WHERE closed_at IS NULL), so it earns the
|
||||
# LIMIT rather than sorting every open role first. Left to the
|
||||
# planner, sqlite3's bundled build (no STAT4) reads the near-100%
|
||||
# selectivity of `closed_at IS NULL` from idx_roles_closed's plain
|
||||
# average and picks that instead, sorting the lot — checked with
|
||||
# EXPLAIN QUERY PLAN on a copy of the live database.
|
||||
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
|
||||
found = [ScoredRole.from_row(r) for r in self._rows(
|
||||
f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?", [*params, limit, offset])]
|
||||
by_uid = self._tags_by_uid([r.uid for r in found])
|
||||
@@ -521,7 +540,7 @@ class Store:
|
||||
where.append("remote = ?")
|
||||
params.append(remote)
|
||||
if days is not None:
|
||||
where.append("first_seen >= ?")
|
||||
where.append("posted_at >= ?")
|
||||
params.append(_iso(self.clock() - timedelta(days=days)))
|
||||
for word in (q or "").split():
|
||||
where.append("title LIKE ? ESCAPE '\\'")
|
||||
@@ -641,7 +660,7 @@ class Store:
|
||||
return (f"""r.uid IN (SELECT uid FROM (
|
||||
SELECT s.uid, s.score, a.limits, ROW_NUMBER() OVER (
|
||||
PARTITION BY s.agent_id
|
||||
ORDER BY s.score DESC, top.first_seen DESC, s.uid) AS n
|
||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
||||
FROM scores s
|
||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||
@@ -936,7 +955,7 @@ class Store:
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
select = ", ".join(["r.uid", "r.first_seen",
|
||||
select = ", ".join(["r.uid", "r.posted_at",
|
||||
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
||||
rows = self._rows(
|
||||
f"""SELECT {select} FROM roles r
|
||||
@@ -944,7 +963,7 @@ class Store:
|
||||
ON ac.company_key = r.company_key AND ac.agent_id = ?
|
||||
WHERE r.closed_at IS NULL AND COALESCE(ac.affinity, 0) <> -1""",
|
||||
[*params, agent_id])
|
||||
return [Candidate(uid=r["uid"], first_seen=r["first_seen"],
|
||||
return [Candidate(uid=r["uid"], posted_at=r["posted_at"],
|
||||
affinity=int(r["affinity"]),
|
||||
gates=[int(r[f"g{n}"]) for n in range(len(gates))])
|
||||
for r in rows]
|
||||
|
||||
Reference in new issue
Block a user