Merge branch 'r2-backend'

# Conflicts:
#	app/storage/schema.py
This commit is contained in:
Wolf committed 2026-09-29 15:22:03 +02:00
commit 562fa3ab09
15 files changed
+924 -157

No files matched your search

+32 -1
View File
@@ -17,7 +17,7 @@ import logging
import sqlite3
from collections.abc import Callable
from app.models import posted_at, posted_host
from app.models import country_of, posted_at, posted_host
log = logging.getLogger(__name__)
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
"could not be normalised into salary_min/salary_max/currency", """
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
"""),
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
def _recompute_country(conn: sqlite3.Connection) -> None:
"""Migration 13's step. `app.models.country_of` used to read a US state
code over a country's ("Bogota, CO" was US), return '' for a line naming
several European countries, and miss most countries outside Europe;
`Store.backfill_from_place` fills only an empty column, so a wrong code
would stay. Recomputed from `place` for every role `extract` has not
read (its answer only ever filled an empty column, but a role it read is
left as it is) and whose board sent no `country` of its own in `raw`
(the `search` plugin's Jobindex and Adzuna rows do), which is what
`Store.upsert_role` would write for that role today."""
rows = conn.execute(
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
updates = []
for uid, place, raw, old in rows:
try:
given = json.loads(raw or "{}").get("country") or ""
except (TypeError, ValueError, AttributeError):
given = ""
if given:
continue
new = country_of(place or "")
if new != old:
updates.append((new, uid))
if updates:
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
len(updates), len(rows))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+15 -2
View File
@@ -1094,11 +1094,24 @@ class Store:
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
favourite. 0 deletes the row, since no row means neutral. Returns the
company with this agent's affinity. Raises ValueError for another
value and KeyError for an unknown company or agent."""
value and KeyError for an unknown company or agent.
A key with roles and no company row, an employer known only from a
search, gets a row here holding only its key and the name its roles
give it, the most frequent one: nothing else can be judged without
one. It has no board, `ats`, `careers_url` or `domain`, which is how
the Companies list tells it from a followed company."""
_check_affinity(affinity)
with self.transaction() as conn:
self._known(conn, "companies", "key", key)
self._known(conn, "agents", "id", agent_id)
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
named = conn.execute(
"""SELECT company FROM roles WHERE company_key=? AND company<>''
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
LIMIT 1""", (key,)).fetchone()
if named is None:
raise KeyError(key)
self._insert_company(conn, key, named[0], {})
if affinity == 0:
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
(agent_id, key))