Merge branch 'r2-backend'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
562fa3ab09
15 files changed
+924
-157
No files matched your search
+32
-1
@@ -17,7 +17,7 @@ import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.models import posted_at, posted_host
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"could not be normalised into salary_min/salary_max/currency", """
|
||||
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
|
||||
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def _recompute_country(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 13's step. `app.models.country_of` used to read a US state
|
||||
code over a country's ("Bogota, CO" was US), return '' for a line naming
|
||||
several European countries, and miss most countries outside Europe;
|
||||
`Store.backfill_from_place` fills only an empty column, so a wrong code
|
||||
would stay. Recomputed from `place` for every role `extract` has not
|
||||
read (its answer only ever filled an empty column, but a role it read is
|
||||
left as it is) and whose board sent no `country` of its own in `raw`
|
||||
(the `search` plugin's Jobindex and Adzuna rows do), which is what
|
||||
`Store.upsert_role` would write for that role today."""
|
||||
rows = conn.execute(
|
||||
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
|
||||
updates = []
|
||||
for uid, place, raw, old in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}").get("country") or ""
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
given = ""
|
||||
if given:
|
||||
continue
|
||||
new = country_of(place or "")
|
||||
if new != old:
|
||||
updates.append((new, uid))
|
||||
if updates:
|
||||
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
|
||||
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
Reference in new issue
Block a user