Migration 19 adds places, place_aliases, place_includes (seeded from the dictionaries in app/models.py and app/places.py, Europe as a continent over the European countries) and role_places, one row per place of a role, resolved by app.places.Gazetteer, NULL when unknown. upsert_role writes them from raw["places"], else the place line cut at ";"; the migration builds them from each role's place column. The Place filter and the country and place gates are joins over the tree: country:DK is any place under Denmark, region:skane any town in Skane, a country gate is met by any place in its countries, and the EU fallback of the word matcher is no longer used. A place: token the gazetteer does not know still matches whole words. Store gains gazetteer, role_places, unresolved_places, add_place_alias, add_place and relink_places. roles.teaser is kept-filled and embedded while there is no description; a vector is dropped when a better text arrives. seniority_model, _stated and _title sit behind roles.seniority, which stays the first filled of them. TRUSTED_POSTED_HOSTS gives way to trusted_posted(host), which reads READS.fields.posted and falls back to the old set for undeclared readers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1205 lines
46 KiB
Python
1205 lines
46 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
# ---------------------------------------------------------------- geography
|
|
|
|
#: Letters NFKD leaves whole because they are letters of their own rather
|
|
#: than a base letter with a mark, spelt the way boards write them in ASCII.
|
|
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
|
|
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
|
|
|
|
|
|
def _place_fold(text: str) -> str:
|
|
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
|
|
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
|
|
since it also builds `company_key`, the primary key of `companies`."""
|
|
return _fold((text or "").casefold().translate(_LETTERS))
|
|
|
|
|
|
#: Every ISO 3166-1 country by its English name, plus the names boards also
|
|
#: write for it: its own language's for a European one, and the usual short
|
|
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
|
|
_COUNTRY_NAMES = """
|
|
AD Andorra
|
|
AE United Arab Emirates | UAE | U.A.E. | Emirates
|
|
AF Afghanistan
|
|
AG Antigua and Barbuda
|
|
AI Anguilla
|
|
AL Albania | Shqipëria | Shqipëri
|
|
AM Armenia | Hayastan
|
|
AO Angola
|
|
AQ Antarctica
|
|
AR Argentina
|
|
AS American Samoa
|
|
AT Austria | Österreich
|
|
AU Australia
|
|
AW Aruba
|
|
AX Åland Islands | Åland
|
|
AZ Azerbaijan | Azərbaycan
|
|
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
|
|
BB Barbados
|
|
BD Bangladesh
|
|
BE Belgium | België | Belgique | Belgien
|
|
BF Burkina Faso
|
|
BG Bulgaria | България | Balgariya
|
|
BH Bahrain
|
|
BI Burundi
|
|
BJ Benin
|
|
BL Saint Barthélemy
|
|
BM Bermuda
|
|
BN Brunei | Brunei Darussalam
|
|
BO Bolivia
|
|
BQ Caribbean Netherlands | Bonaire
|
|
BR Brazil | Brasil
|
|
BS Bahamas
|
|
BT Bhutan
|
|
BV Bouvet Island
|
|
BW Botswana
|
|
BY Belarus | Беларусь | Belarus'
|
|
BZ Belize
|
|
CA Canada
|
|
CC Cocos Islands | Cocos (Keeling) Islands
|
|
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
|
|
CF Central African Republic
|
|
CG Republic of the Congo | Congo-Brazzaville
|
|
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
|
|
CI Côte d'Ivoire | Ivory Coast
|
|
CK Cook Islands
|
|
CL Chile
|
|
CM Cameroon
|
|
CN China | People's Republic of China
|
|
CO Colombia
|
|
CR Costa Rica
|
|
CU Cuba
|
|
CV Cabo Verde | Cape Verde
|
|
CW Curaçao
|
|
CX Christmas Island
|
|
CY Cyprus | Κύπρος | Kypros | Kibris
|
|
CZ Czechia | Czech Republic | Česko | Česká republika
|
|
DE Germany | Deutschland
|
|
DJ Djibouti
|
|
DK Denmark | Danmark
|
|
DM Dominica
|
|
DO Dominican Republic
|
|
DZ Algeria
|
|
EC Ecuador
|
|
EE Estonia | Eesti
|
|
EG Egypt
|
|
EH Western Sahara
|
|
ER Eritrea
|
|
ES Spain | España | Espanya
|
|
ET Ethiopia
|
|
FI Finland | Suomi
|
|
FJ Fiji
|
|
FK Falkland Islands
|
|
FM Micronesia
|
|
FO Faroe Islands | Føroyar | Færøerne
|
|
FR France
|
|
GA Gabon
|
|
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
|
|
GD Grenada
|
|
GE Georgia | Sakartvelo
|
|
GF French Guiana
|
|
GG Guernsey
|
|
GH Ghana
|
|
GI Gibraltar
|
|
GL Greenland
|
|
GM Gambia
|
|
GN Guinea
|
|
GP Guadeloupe
|
|
GQ Equatorial Guinea
|
|
GR Greece | Ελλάδα | Hellas | Ellada
|
|
GS South Georgia and the South Sandwich Islands
|
|
GT Guatemala
|
|
GU Guam
|
|
GW Guinea-Bissau
|
|
GY Guyana
|
|
HK Hong Kong
|
|
HM Heard Island and McDonald Islands
|
|
HN Honduras
|
|
HR Croatia | Hrvatska
|
|
HT Haiti
|
|
HU Hungary | Magyarország
|
|
ID Indonesia
|
|
IE Ireland | Éire
|
|
IL Israel
|
|
IM Isle of Man
|
|
IN India
|
|
IO British Indian Ocean Territory
|
|
IQ Iraq
|
|
IR Iran
|
|
IS Iceland | Ísland
|
|
IT Italy | Italia
|
|
JE Jersey
|
|
JM Jamaica
|
|
JO Jordan
|
|
JP Japan
|
|
KE Kenya
|
|
KG Kyrgyzstan
|
|
KH Cambodia
|
|
KI Kiribati
|
|
KM Comoros
|
|
KN Saint Kitts and Nevis
|
|
KP North Korea
|
|
KR South Korea | Korea | Republic of Korea
|
|
KW Kuwait
|
|
KY Cayman Islands
|
|
KZ Kazakhstan
|
|
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
|
|
LB Lebanon
|
|
LC Saint Lucia
|
|
LI Liechtenstein
|
|
LK Sri Lanka
|
|
LR Liberia
|
|
LS Lesotho
|
|
LT Lithuania | Lietuva
|
|
LU Luxembourg | Lëtzebuerg | Luxemburg
|
|
LV Latvia | Latvija
|
|
LY Libya
|
|
MA Morocco
|
|
MC Monaco
|
|
MD Moldova | Republic of Moldova
|
|
ME Montenegro | Crna Gora
|
|
MF Saint Martin
|
|
MG Madagascar
|
|
MH Marshall Islands
|
|
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
|
|
ML Mali
|
|
MM Myanmar | Burma
|
|
MN Mongolia
|
|
MO Macao | Macau
|
|
MP Northern Mariana Islands
|
|
MQ Martinique
|
|
MR Mauritania
|
|
MS Montserrat
|
|
MT Malta
|
|
MU Mauritius
|
|
MV Maldives
|
|
MW Malawi
|
|
MX Mexico | México
|
|
MY Malaysia
|
|
MZ Mozambique
|
|
NA Namibia
|
|
NC New Caledonia
|
|
NE Niger
|
|
NF Norfolk Island
|
|
NG Nigeria
|
|
NI Nicaragua
|
|
NL Netherlands | The Netherlands | Nederland | Holland
|
|
NO Norway | Norge | Noreg
|
|
NP Nepal
|
|
NR Nauru
|
|
NU Niue
|
|
NZ New Zealand | Aotearoa
|
|
OM Oman
|
|
PA Panama | Panamá
|
|
PE Peru | Perú
|
|
PF French Polynesia
|
|
PG Papua New Guinea
|
|
PH Philippines
|
|
PK Pakistan
|
|
PL Poland | Polska
|
|
PM Saint Pierre and Miquelon
|
|
PN Pitcairn Islands
|
|
PR Puerto Rico
|
|
PS Palestine
|
|
PT Portugal
|
|
PW Palau
|
|
PY Paraguay
|
|
QA Qatar
|
|
RE Réunion
|
|
RO Romania | România
|
|
RS Serbia | Srbija | Србија
|
|
RU Russia | Russian Federation | Россия | Rossiya
|
|
RW Rwanda
|
|
SA Saudi Arabia | KSA
|
|
SB Solomon Islands
|
|
SC Seychelles
|
|
SD Sudan
|
|
SE Sweden | Sverige
|
|
SG Singapore
|
|
SH Saint Helena
|
|
SI Slovenia | Slovenija
|
|
SJ Svalbard and Jan Mayen | Svalbard
|
|
SK Slovakia | Slovensko
|
|
SL Sierra Leone
|
|
SM San Marino
|
|
SN Senegal
|
|
SO Somalia
|
|
SR Suriname
|
|
SS South Sudan
|
|
ST São Tomé and Príncipe
|
|
SV El Salvador
|
|
SX Sint Maarten
|
|
SY Syria
|
|
SZ Eswatini | Swaziland
|
|
TC Turks and Caicos Islands
|
|
TD Chad
|
|
TF French Southern Territories
|
|
TG Togo
|
|
TH Thailand
|
|
TJ Tajikistan
|
|
TK Tokelau
|
|
TL Timor-Leste | East Timor
|
|
TM Turkmenistan
|
|
TN Tunisia
|
|
TO Tonga
|
|
TR Turkey | Türkiye
|
|
TT Trinidad and Tobago
|
|
TV Tuvalu
|
|
TW Taiwan
|
|
TZ Tanzania
|
|
UA Ukraine | Україна | Ukraina
|
|
UG Uganda
|
|
UM United States Minor Outlying Islands
|
|
US United States | United States of America | USA | U.S.A. | US | U.S.
|
|
UY Uruguay
|
|
UZ Uzbekistan
|
|
VA Vatican City | Holy See
|
|
VC Saint Vincent and the Grenadines
|
|
VE Venezuela
|
|
VG British Virgin Islands
|
|
VI U.S. Virgin Islands | US Virgin Islands
|
|
VN Vietnam | Viet Nam
|
|
VU Vanuatu
|
|
WF Wallis and Futuna
|
|
WS Samoa
|
|
XK Kosovo | Kosova
|
|
YE Yemen
|
|
YT Mayotte
|
|
ZA South Africa
|
|
ZM Zambia
|
|
ZW Zimbabwe
|
|
"""
|
|
|
|
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
|
|
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
|
|
|
|
#: The countries `EU` stands for when a line names several: the UN
|
|
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
|
|
#: the Council of Europe's members in the Caucasus and Turkey, which
|
|
#: European employers list beside their other offices.
|
|
_EUROPE = frozenset((
|
|
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
|
|
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
|
|
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
|
|
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
|
|
"XK",
|
|
))
|
|
_EUROPE_OR_EU = _EUROPE | {"EU"}
|
|
|
|
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
|
|
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
|
|
|
#: The cities boards name most, and a few regions below a country. A US
|
|
#: state or a Canadian province in the same part of the line beats one of
|
|
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
|
_CITIES: dict[str, str] = {
|
|
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
|
|
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
|
|
"luik"), "BE"),
|
|
**dict.fromkeys(("sofia",), "BG"),
|
|
**dict.fromkeys(("zagreb",), "HR"),
|
|
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
|
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
|
|
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
|
|
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
|
|
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
|
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
|
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
|
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
|
|
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
|
"lille", "rennes", "amiens"), "FR"),
|
|
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
|
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
|
|
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
|
|
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
|
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
|
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
|
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
|
|
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
|
**dict.fromkeys(("budapest",), "HU"),
|
|
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
|
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
|
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
|
|
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
|
|
**dict.fromkeys(("riga",), "LV"),
|
|
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
|
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
|
|
**dict.fromkeys(("valletta",), "MT"),
|
|
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
|
"delft"), "NL"),
|
|
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
|
"PL"),
|
|
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
|
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
|
|
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
|
**dict.fromkeys(("bratislava",), "SK"),
|
|
**dict.fromkeys(("ljubljana",), "SI"),
|
|
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
|
"granada", "santiago de compostela"), "ES"),
|
|
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
|
|
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
|
|
**dict.fromkeys(("reykjavik",), "IS"),
|
|
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
|
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
|
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
|
|
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
|
|
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
|
|
"slough", "southampton", "swindon", "liverpool", "redhill",
|
|
"milton keynes"), "GB"),
|
|
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
|
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
|
|
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
|
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
|
**dict.fromkeys(("tirana",), "AL"),
|
|
**dict.fromkeys(("yerevan",), "AM"),
|
|
**dict.fromkeys(("tbilisi",), "GE"),
|
|
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
|
|
"boston", "chicago", "austin", "los angeles", "palo alto",
|
|
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
|
|
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
|
|
"washington dc", "washington d c", "miami", "dallas", "houston",
|
|
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
|
|
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
|
|
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
|
|
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
|
|
"CA"),
|
|
**dict.fromkeys(("buenos aires",), "AR"),
|
|
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
|
|
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
|
|
**dict.fromkeys(("bogota",), "CO"),
|
|
**dict.fromkeys(("lima", "miraflores"), "PE"),
|
|
**dict.fromkeys(("montevideo",), "UY"),
|
|
**dict.fromkeys(("managua",), "NI"),
|
|
**dict.fromkeys(("santo domingo",), "DO"),
|
|
**dict.fromkeys(("san salvador",), "SV"),
|
|
**dict.fromkeys(("panama city",), "PA"),
|
|
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
|
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
|
|
**dict.fromkeys(("riyadh",), "SA"),
|
|
**dict.fromkeys(("doha",), "QA"),
|
|
**dict.fromkeys(("manama",), "BH"),
|
|
**dict.fromkeys(("salmiya",), "KW"),
|
|
**dict.fromkeys(("muscat",), "OM"),
|
|
**dict.fromkeys(("cairo", "maadi"), "EG"),
|
|
**dict.fromkeys(("casablanca",), "MA"),
|
|
**dict.fromkeys(("dakar",), "SN"),
|
|
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
|
**dict.fromkeys(("almaty",), "KZ"),
|
|
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
|
|
**dict.fromkeys(("dhaka",), "BD"),
|
|
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
|
|
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
|
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
|
|
**dict.fromkeys(("singapore",), "SG"),
|
|
**dict.fromkeys(("manila", "taguig"), "PH"),
|
|
**dict.fromkeys(("bangkok",), "TH"),
|
|
**dict.fromkeys(("phnom penh",), "KH"),
|
|
**dict.fromkeys(("taipei",), "TW"),
|
|
**dict.fromkeys(("tokyo", "osaka"), "JP"),
|
|
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
|
|
**dict.fromkeys(("seoul",), "KR"),
|
|
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
|
|
"queensland"), "AU"),
|
|
}
|
|
|
|
#: US states and Canadian provinces, by name and by the code boards write
|
|
#: after a comma ("Boston, MA", "Toronto, ON").
|
|
_SUBDIVISIONS = """
|
|
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
|
|
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
|
|
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
|
|
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
|
|
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
|
|
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
|
|
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
|
|
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
|
|
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
|
|
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
|
|
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
|
|
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
|
|
CA Nunavut NU
|
|
"""
|
|
|
|
|
|
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
|
|
names: dict[str, str] = {}
|
|
codes: dict[str, str] = {}
|
|
for line in _SUBDIVISIONS.split("\n"):
|
|
if line:
|
|
country, rest = line.split(" ", 1)
|
|
for entry in rest.split("|"):
|
|
name, code = entry.strip().rsplit(" ", 1)
|
|
names[_place_fold(name)] = country
|
|
codes[code] = country
|
|
return names, codes
|
|
|
|
|
|
_STATE_NAMES, _STATE_CODES = _subdivisions()
|
|
|
|
|
|
def _names() -> dict[str, tuple[str, str]]:
|
|
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
|
|
`state`. A name that is a city and a state ("New York") is the city, so
|
|
it does not beat another city in its line. A name that is a country and
|
|
a state ("Georgia") is the country here and read as the state only when
|
|
the line names a place in the US too."""
|
|
names: dict[str, tuple[str, str]] = {}
|
|
for name, code in _STATE_NAMES.items():
|
|
names[name] = (code, "state")
|
|
for name, code in _CITIES.items():
|
|
names[_place_fold(name)] = (code, "city")
|
|
for line in _COUNTRY_NAMES.split("\n"):
|
|
if line:
|
|
code, rest = line.split(" ", 1)
|
|
for name in rest.split("|"):
|
|
names[_place_fold(name)] = (code, "country")
|
|
for name in _REGIONS:
|
|
names[name] = ("EU", "country")
|
|
return names
|
|
|
|
|
|
_NAMES = _names()
|
|
|
|
#: A state name that is also a country's, read as the state only when the
|
|
#: line names a place in that state's country too.
|
|
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
|
|
if _NAMES[name][1] == "country"}
|
|
|
|
#: The longest name in words, so a lookup tries "new york city" before "new
|
|
#: york" and "northern ireland" before "ireland".
|
|
_LONGEST = max(len(name.split()) for name in _NAMES)
|
|
|
|
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
|
|
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
|
|
|
|
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
|
|
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
|
|
|
|
#: Where one location in a line ends and the next begins. A state beats a
|
|
#: city only within one: "London; Ontario" is two places.
|
|
_SEGMENT = re.compile(r"[;|/\n]")
|
|
|
|
|
|
def _hits(folded: str) -> list[tuple[str, str, str]]:
|
|
"""The names in a folded line, longest first at each word, as
|
|
`(name, code, kind)`."""
|
|
words = folded.split()
|
|
hits = []
|
|
i = 0
|
|
while i < len(words):
|
|
for n in range(min(_LONGEST, len(words) - i), 0, -1):
|
|
name = " ".join(words[i:i + n])
|
|
if name in _NAMES:
|
|
hits.append((name, *_NAMES[name]))
|
|
i += n
|
|
break
|
|
else:
|
|
i += 1
|
|
return hits
|
|
|
|
|
|
def country_of(place: str) -> str:
|
|
"""The one country a board's location line names, as an ISO code; `EU`
|
|
when it names several and all of them are in Europe; '' when it names
|
|
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
|
|
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
|
|
New York" and "Remote, EU or US" are ''.
|
|
|
|
Within one location (the line split at `;`, `|` and `/`) the widest unit
|
|
named wins: a country beats a state or a province, which beats a city.
|
|
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
|
|
country also beats the region it is in: "London, EMEA" is GB.
|
|
|
|
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
|
|
count as the state only when the line names a place in the US too:
|
|
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
|
|
than Delaware. A country code in parentheses, "Leicester (GB)", is read
|
|
only when no name matched."""
|
|
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
|
|
named = {code for _, hits in segments for _, code, _ in hits}
|
|
found: set[str] = set()
|
|
for text, hits in segments:
|
|
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
|
|
countries = {code for name, code, kind in hits
|
|
if kind == "country" and code != "EU" and name not in shared}
|
|
states = {code for _, code, kind in hits if kind == "state"}
|
|
states |= {_SHARED_NAMES[name] for name in shared}
|
|
states |= {country for code in _STATE_CODE.findall(text)
|
|
if (country := _STATE_CODES.get(code))
|
|
and (code not in _ISO or country in named)}
|
|
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
|
|
found |= {code for _, code, _ in hits if code == "EU"}
|
|
if not found:
|
|
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
|
|
if len(found) == 1:
|
|
return found.pop()
|
|
if found and found <= _EUROPE_OR_EU:
|
|
countries = found - {"EU"}
|
|
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
|
|
return ""
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
#: Words in a location line that say how the role is worked, for a board that
|
|
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
|
|
#: "Distributed" as the location and nothing else.
|
|
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
|
|
("hybrid", "hybrid"),
|
|
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
|
|
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
|
|
("worldwide", "remote"), ("telecommute", "remote"),
|
|
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
|
|
("in-office", "onsite"), ("in office", "onsite"),
|
|
)
|
|
|
|
|
|
def remote_of(place: str) -> str:
|
|
"""How a board's location line says the role is worked, as one of
|
|
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
|
|
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
|
|
and a line that names only a place is ''."""
|
|
folded = _fold(place or "")
|
|
for word, mode in _REMOTE_WORDS:
|
|
if word in folded:
|
|
return mode
|
|
return ""
|
|
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
|
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips the `any` gates).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
#: Why a company is on the list checked by hand. `unread` is a host the app
|
|
#: could read and has no parser for yet, `robots` a site whose robots.txt
|
|
#: disallows it, `blocked` one that answers a program with a challenge, `key`
|
|
#: one whose API answers only with a key lifted from its own script, `group` a
|
|
#: company that sits inside its owner's board, `gone` one whose site is dead,
|
|
#: and `added` one put there by hand or by an import and not looked at. ''
|
|
#: is none of these: the company is read, or is not a matter for the list.
|
|
HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"]
|
|
HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
|
|
|
|
|
# -------------------------------------------------------------- posting dates
|
|
|
|
#: Hosts whose date is a creation or first-publication timestamp, for a
|
|
#: reader that does not declare `READS` yet (see `trusted_posted`); a reader
|
|
#: that declares it says so with `fields.posted="trusted"`. Checked against
|
|
#: live boards on 2026-09-29:
|
|
#:
|
|
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
|
|
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
|
#: with `first_published` five months before `updated_at`, proving the
|
|
#: first stays fixed while the second moves on an edit — the resettable
|
|
#: field decisions.md used to warn against is `updated_at`, which this
|
|
#: reads only when `first_published` is missing (rare: 0 of 208 on that
|
|
#: board). The large real gaps this migration measured for Greenhouse
|
|
#: roles (average 143 days between this date and `first_seen`) are further
|
|
#: evidence it holds still rather than tracking every edit.
|
|
#: - `lever` (`createdAt`), `ashby` (`publishedAt`), `personio` (`createdAt`):
|
|
#: named and shaped as creation dates, with no "last edited" field in the
|
|
#: feed at all to reset from, and the same large real gaps as Greenhouse's.
|
|
#: - `smartrecruiters` (`releasedDate`), `recruitee` (`published_at`, falling
|
|
#: back to `created_at`): `releasedDate` reads the same way; a live
|
|
#: Recruitee board (bunq) showed `published_at` and `created_at` minutes
|
|
#: apart while `updated_at` had moved days later on an edited posting,
|
|
#: confirming the field this migration reads is not the one that moves.
|
|
#: - `workable` (`published`): named as a publish date, not an edit one, but
|
|
#: only one live board had postings to check (huggingface, 8 roles) and
|
|
#: none showed a second, "last touched" field to compare against, so this
|
|
#: is trusted on the name and shape alone.
|
|
#: - `jobindex`: the RSS `pubDate` a saved search returns. The live
|
|
#: database's 158 Jobindex-found roles have a `published_at`/`first_seen`
|
|
#: gap of 0.4 to 31 days (median 12), which reads as real, spread-out
|
|
#: posting ages rather than a date that jumps to "now" on every re-crawl.
|
|
#: - `adzuna`: `created`, an aggregator's own record-creation stamp — named
|
|
#: as creation, not modification, like the rest. No live check: this
|
|
#: deployment carries no Adzuna key (`plugins.search`'s own docstring:
|
|
#: "not yet run live") and no Adzuna subscription exists to fetch.
|
|
#:
|
|
#: - `teamtailor`: the feed's `pubDate`, and `platsbanken`: `publication_date`,
|
|
#: both named as publication. Read on live feeds on 2026-09-29, where the
|
|
#: dates were spread over weeks, not checked against stored roles.
|
|
#:
|
|
#: - `workday`: the role's `startDate`, which matched the listing's "Posted N
|
|
#: Days Ago" on twelve tenants; `successfactors`: the role page's
|
|
#: `datePosted`; `oracle`: `ExternalPostedStartDate`; `phenom`: the role
|
|
#: page's JSON-LD `datePosted`, which is earlier than the listing's date
|
|
#: where the two differ; `bamboohr`: `datePosted`; `jobylon`: the role
|
|
#: page's JSON-LD date; `rippling`: `createdOn`. Read on live boards on
|
|
#: 2026-09-29 by the builders of those modules, not checked against stored
|
|
#: roles. The other hosts added that day (Eightfold, Jibe, Gem, Deel, HiBob,
|
|
#: HR Manager) state a date nobody checked, and Pinpoint states none.
|
|
#:
|
|
#: A host not listed here, or a role whose stored `published_at` fails the
|
|
#: sanity check in `posted_at`, keeps `first_seen` as its posted date.
|
|
_UNDECLARED_TRUSTED: frozenset[str] = frozenset({
|
|
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
|
"personio", "teamtailor", "jobindex", "adzuna", "platsbanken",
|
|
"workday", "successfactors", "oracle", "phenom", "bamboohr", "jobylon", "rippling",
|
|
})
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _declared_posted() -> dict[str, str]:
|
|
"""`fields.posted` of every reader that declares `READS`, by name. Read
|
|
once: the readers are loaded when `plugins.readers` is imported. `plugins`
|
|
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
|
|
reaches it."""
|
|
root = str(Path(__file__).resolve().parents[1])
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
from plugins.readers import READERS
|
|
|
|
return {name: module.READS.fields.posted for name, module in READERS.items()
|
|
if getattr(module, "READS", None) is not None}
|
|
|
|
|
|
def trusted_posted(host: str) -> bool:
|
|
"""Whether `host` states a date that is safe to read as when a role was
|
|
posted: its reader declares `fields.posted="trusted"`, or, for a reader
|
|
without `READS` (or a search provider not yet a reader), it is in
|
|
`_UNDECLARED_TRUSTED`."""
|
|
declared = _declared_posted()
|
|
if host in declared:
|
|
return declared[host] == "trusted"
|
|
return host in _UNDECLARED_TRUSTED
|
|
|
|
|
|
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
|
#: real posting date: no board here existed before 2015.
|
|
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
|
|
|
|
|
def posted_host(raw: dict[str, Any]) -> str:
|
|
"""The host a role's dates came from, read the way `ats` and `search`
|
|
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
|
`lever`, ...), `provider` for a cross-company search (`jobindex`,
|
|
`adzuna`). '' when neither plugin wrote one."""
|
|
return str(raw.get("board") or raw.get("provider") or "")
|
|
|
|
|
|
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
|
"""The date a role counts as posted: the earlier of `first_seen` (when
|
|
the app itself first stored it) and `published_at` (what the board
|
|
claims), but only when `trusted_posted(host)` and
|
|
`published_at` is neither absurdly old nor later than `first_seen` — a
|
|
role cannot be posted after the app already saw it, trusted host or not,
|
|
which also catches a `published_at` the board reports in the future
|
|
(`first_seen` is never later than the moment this runs). Both dates are
|
|
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
|
is exact and needs no parsing. A role with no trusted date, or with one
|
|
that fails the check, keeps `first_seen`."""
|
|
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
|
|
return published_at
|
|
return first_seen
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class RoleCopy(Model):
|
|
"""One open copy of a kept role, as the reader lists it under "Also
|
|
posted": where it is listed (`host`, the search provider or the board's
|
|
ATS) and the link that opens it there."""
|
|
|
|
uid: str
|
|
host: str = ""
|
|
place: str = ""
|
|
original_url: str = ""
|
|
posted_at: str = ""
|
|
|
|
|
|
class RolePlace(Model):
|
|
"""One place of a role (`role_places`): the host's `text`, and the place
|
|
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
|
|
empty) while it resolves to none. `url` is the link of a copy folded into
|
|
the role, '' for the role's own places."""
|
|
|
|
position: int = 0
|
|
text: str = ""
|
|
place_id: int | None = None
|
|
name: str = ""
|
|
kind: str = ""
|
|
url: str = ""
|
|
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
#: The pay a board stated, as one human-readable line, whether or not it
|
|
#: could be normalised into `salary_min`/`salary_max`/`currency`
|
|
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
|
|
salary_text: str = ""
|
|
#: The level shown and filtered on: the first filled of `seniority_model`,
|
|
#: `seniority_stated` and `seniority_title`.
|
|
seniority: str = ""
|
|
seniority_model: str = ""
|
|
seniority_stated: str = ""
|
|
seniority_title: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
|
languages: JsonStrings = []
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
#: The short text the host's list carries, until `description` arrives.
|
|
teaser: str = ""
|
|
original_url: str = ""
|
|
published_at: str
|
|
#: The date this role counts as posted: see `posted_at`. Used for
|
|
#: `recency` and the feed's newest order; `first_seen` stays what it
|
|
#: always was, when the app itself first stored the role.
|
|
posted_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
#: The vote of the agent the role was read for (see `role_votes`): None
|
|
#: unjudged, else -1 pass, 0 neutral, 1 like.
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
#: NULL unread, else when the feed opened it or judged it. A vote stamps
|
|
#: this too, see `Store.set_vote`: voting is reading.
|
|
read_at: str | None = None
|
|
summary: str = ""
|
|
summarised_at: str | None = None
|
|
#: The posting page as plain text, for a role whose description is a teaser.
|
|
article_text: str = ""
|
|
article_at: str | None = None
|
|
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
|
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
|
tags: list[str] = []
|
|
#: The uid of the kept role when this one is a copy of the same posting
|
|
#: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of
|
|
#: the feed and the counts but still opens by uid.
|
|
dup_of: str | None = None
|
|
#: The open copies of this role, oldest first, when it is the kept one;
|
|
#: empty otherwise. Not a column: filled by `Store.role`.
|
|
copies: list[RoleCopy] = []
|
|
#: Every place of the role, in the host's order. Not a column: filled by
|
|
#: `Store.role` from `role_places`.
|
|
places: list[RolePlace] = []
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def salary_state(self) -> Literal["normalised", "stated", ""]:
|
|
"""Not a column: 'normalised' when `salary_min` or `salary_max` is
|
|
set, 'stated' when only `salary_text` is, '' otherwise. Gates and
|
|
scoring read `salary_min` alone and never this (docs/decisions.md);
|
|
it is what the feed's Pay filter and badge read instead
|
|
(`Store.PAY`, `static/roles.js`)."""
|
|
if self.salary_min is not None or self.salary_max is not None:
|
|
return "normalised"
|
|
if self.salary_text:
|
|
return "stated"
|
|
return ""
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See app/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class ScoredRole(Role):
|
|
"""A role as `/api/roles/{uid}` serves it: with one agent's score and
|
|
gate reason when an agent was asked for, and without otherwise."""
|
|
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
#: The agent's prompt that gave the role its fit; "" when fit did not
|
|
#: count (gated, no vector, no prompt with text).
|
|
prompt: str = ""
|
|
|
|
|
|
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
|
|
#: cap, built once here rather than by the client, so a list response never
|
|
#: carries more of a role's text than a row draws.
|
|
EXCERPT_LIMIT = 200
|
|
|
|
|
|
def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str:
|
|
"""`text` (a role's summary, or the opening of its description) with its
|
|
whitespace collapsed, cut to `limit` characters at a word boundary with
|
|
an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a
|
|
`SUBSTR` of the source column rather than the whole of it, so the feed's
|
|
list never carries what only the reader needs."""
|
|
collapsed = " ".join((text or "").split())
|
|
if len(collapsed) <= limit:
|
|
return collapsed
|
|
cut = collapsed[:limit]
|
|
space = cut.rfind(" ")
|
|
trimmed = cut[:space] if space > limit * 0.6 else cut
|
|
return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…"
|
|
|
|
|
|
class RoleRow(Model):
|
|
"""One row of the feed: what `GET /api/roles` serves per role, the
|
|
fields a row draws and nothing else — never `description`, `feed_text`,
|
|
`raw` or `article_text`, the four columns docs/decisions.md measures at
|
|
about 5.5 KB a role between them. The reader fetches the rest by uid when
|
|
it opens a role (`GET /api/roles/{uid}`, the full `Role`).
|
|
|
|
`excerpt` is built on the server (see `excerpt`, above): the summary when
|
|
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
|
|
characters. The pay fields are what the row's badge reads: the three
|
|
normalised columns, the line the board stated, and `salary_state`,
|
|
derived as on `Role`. `original_url` is one short string, kept so the
|
|
`o` key opens a selected row's posting without opening the reader.
|
|
`seniority` is the title's level until extract reads the posting, ''
|
|
when neither says. `copies` counts the open copies the feed hides behind
|
|
this row."""
|
|
|
|
uid: str
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
place: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
function: str = ""
|
|
seniority: str = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
salary_text: str = ""
|
|
original_url: str = ""
|
|
posted_at: str = ""
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
prompt: str = ""
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
read_at: str | None = None
|
|
tags: list[str] = []
|
|
excerpt: str = ""
|
|
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
|
|
copies: int = 0
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def salary_state(self) -> str:
|
|
"""As `Role.salary_state`: 'normalised', 'stated' or ''."""
|
|
if self.salary_min is not None or self.salary_max is not None:
|
|
return "normalised"
|
|
return "stated" if self.salary_text else ""
|
|
|
|
|
|
class Company(Model):
|
|
"""A row of `companies`, with `affinity` as the agent it was read for
|
|
has it (see `agent_companies`); 0 when no agent was asked for."""
|
|
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
#: The host the careers site runs on, readable or not; "" when unknown.
|
|
#: A company with an `ats` and no board subscription is one the app
|
|
#: cannot read yet. See `app.jobs.discover`.
|
|
ats: str = ""
|
|
country: str = ""
|
|
city: str = ""
|
|
#: The name `ats` knows the company by, the `acme` of acme.teamtailor.com;
|
|
#: "" where discovery found none. Kept for a host no parser reads yet.
|
|
ats_slug: str = ""
|
|
#: The Watchlist's rating, global and not an agent's: None not yet rated,
|
|
#: 1 worth checking by hand, 0 not.
|
|
watch: int | None = None
|
|
#: When the careers page was last opened from the Watchlist; "" never.
|
|
careers_opened_at: str = ""
|
|
#: Why the company is read by hand, one of `HAND_REASONS`; "" when it is
|
|
#: not on that list.
|
|
hand_reason: str = ""
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
# ------------------------------------------------------------------ agents
|
|
|
|
#: The role columns a gate can name. `company` matches on the company key;
|
|
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
|
|
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
|
|
#: case-insensitive equality, except `salary_min` (at least the value) and
|
|
#: `languages` (every language the role requires is one of the values).
|
|
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
|
"currency", "permit", "languages", "company"]
|
|
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
|
|
|
GateMode = Literal["require", "any", "prefer"]
|
|
|
|
|
|
class Gate(Model):
|
|
"""One condition of an agent. `require` gates a role that fails it (the
|
|
row is kept with the reason); the `any` gates together are one group a
|
|
role must match at least one of ("in Denmark, or remote"); `prefer` adds
|
|
`weight` to its score when it matches. A gate rejects on evidence only: a
|
|
role whose column is empty is not rejected, since `extract` fills the
|
|
column later for the roles that rank, and only for those."""
|
|
|
|
field: GateField
|
|
values: list[str | int | float] = Field(min_length=1)
|
|
mode: GateMode = "require"
|
|
weight: float = Field(1.0, ge=0)
|
|
|
|
|
|
class Weights(Model):
|
|
"""How much each score term counts. Summed with the prefer gates' weights
|
|
into the denominator, so a score stays in 0..1 whatever the numbers."""
|
|
|
|
fit: float = Field(0.6, ge=0)
|
|
recency: float = Field(0.2, ge=0)
|
|
affinity: float = Field(0.2, ge=0)
|
|
#: Cosine to the liked-minus-passed direction of the agent's votes.
|
|
taste: float = Field(0.2, ge=0)
|
|
|
|
|
|
class Limits(Model):
|
|
"""How far down an agent's ranking the two model stages read. Each pair
|
|
is a top N, the backstop, and a minimum score, the real stop: a role
|
|
is read when it is among the agent's best `extract_top` ungated scores
|
|
and not below `extract_min`, and summarised likewise. Fit is a
|
|
calibrated level (`score.scale`), so a bar means the same in every run.
|
|
A fresh role with the worst fit scores about 0.27 on the default
|
|
weights, a perfect one about 0.87. Measured on 16,841 open roles for the
|
|
"Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
|
|
admits about 300 roles to extract and 0.65 about 35 to summarise, the
|
|
long answer; the top N catches an agent whose bar admits far more. An
|
|
agent keeps the limits stored with it; only a key its JSON lacks
|
|
follows these defaults."""
|
|
|
|
extract_top: int = Field(500, ge=0)
|
|
extract_min: float = Field(0.55, ge=0, le=1)
|
|
summarise_top: int = Field(30, ge=0)
|
|
summarise_min: float = Field(0.65, ge=0, le=1)
|
|
|
|
|
|
class Prompt(Model):
|
|
"""One of an agent's descriptions of what it looks for. Each is embedded
|
|
on its own and a role's fit is its best prompt's (see `app.jobs.score`),
|
|
so one agent can hunt several kinds of role without blending them. A
|
|
prompt with no text has no fit to give."""
|
|
|
|
name: str
|
|
text: str = ""
|
|
|
|
@field_validator("name")
|
|
@classmethod
|
|
def _named(cls, name: str) -> str:
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a prompt needs a name")
|
|
return name
|
|
|
|
|
|
def check_prompts(prompts: list[Prompt]) -> list[Prompt]:
|
|
"""The prompts as they are, or ValueError when two share a name
|
|
regardless of case."""
|
|
seen: set[str] = set()
|
|
for prompt in prompts:
|
|
if prompt.name.casefold() in seen:
|
|
raise ValueError(f"two prompts are named {prompt.name!r}")
|
|
seen.add(prompt.name.casefold())
|
|
return prompts
|
|
|
|
|
|
#: An agent's prompts, in order, their names unique regardless of case.
|
|
Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)]
|
|
|
|
|
|
class Agent(Model):
|
|
"""A row of `agents`: a named search with its own prompts, gates, weights
|
|
and limits. The prompts are rows of `agent_prompts`, read alongside; their
|
|
cached vectors are columns this does not declare."""
|
|
|
|
id: int
|
|
name: str
|
|
prompts: Prompts = []
|
|
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
|
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
|
|
enabled: bool = True
|
|
created_at: str
|
|
|
|
|
|
class Candidate(Model):
|
|
"""What scoring reads about one open role for one agent: its posted
|
|
date (for `recency`), its company's affinity for that agent, and per
|
|
gate `1` match, `0` no match, `-1` the column is empty."""
|
|
|
|
uid: str
|
|
posted_at: str
|
|
affinity: int = 0
|
|
gates: list[int] = []
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|