Merge branch 'r2-backend'

# Conflicts:
#	app/storage/schema.py
This commit is contained in:
Wolf committed 2026-09-29 15:22:03 +02:00
commit 562fa3ab09
15 files changed
+924 -157

No files matched your search

+32 -1
View File
@@ -17,7 +17,7 @@ import logging
import sqlite3
from collections.abc import Callable
from app.models import posted_at, posted_host
from app.models import country_of, posted_at, posted_host
log = logging.getLogger(__name__)
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
"could not be normalised into salary_min/salary_max/currency", """
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
"""),
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
def _recompute_country(conn: sqlite3.Connection) -> None:
"""Migration 13's step. `app.models.country_of` used to read a US state
code over a country's ("Bogota, CO" was US), return '' for a line naming
several European countries, and miss most countries outside Europe;
`Store.backfill_from_place` fills only an empty column, so a wrong code
would stay. Recomputed from `place` for every role `extract` has not
read (its answer only ever filled an empty column, but a role it read is
left as it is) and whose board sent no `country` of its own in `raw`
(the `search` plugin's Jobindex and Adzuna rows do), which is what
`Store.upsert_role` would write for that role today."""
rows = conn.execute(
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
updates = []
for uid, place, raw, old in rows:
try:
given = json.loads(raw or "{}").get("country") or ""
except (TypeError, ValueError, AttributeError):
given = ""
if given:
continue
new = country_of(place or "")
if new != old:
updates.append((new, uid))
if updates:
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
len(updates), len(rows))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])