country_of folds with _place_fold (ł, đ, ø, æ map to ASCII; _fold keeps building company_key unchanged), knows every ISO country by English name and European ones by their own, and the cities among the 150 most frequent unresolved places on the live data. Within one location a country beats a US state or Canadian province, which beats a city. A state code that is also an ISO code counts as the state only when the line names a US place. Several countries, all European, are EU; otherwise ''. A code in parentheses is read when no name matched. Migration 12 recomputes country from place for roles extract has not read and whose board sent no country in raw. On the live data: open roles with no country 2,827 -> 1,183 of 16,841; 1,705 open roles change, 48 from a code. Co-Authored-By: Claude Opus <noreply@anthropic.com>
960 lines
35 KiB
Python
960 lines
35 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import BeforeValidator, Field
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
# ---------------------------------------------------------------- geography
|
|
|
|
#: Letters NFKD leaves whole because they are letters of their own rather
|
|
#: than a base letter with a mark, spelt the way boards write them in ASCII.
|
|
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
|
|
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
|
|
|
|
|
|
def _place_fold(text: str) -> str:
|
|
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
|
|
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
|
|
since it also builds `company_key`, the primary key of `companies`."""
|
|
return _fold((text or "").casefold().translate(_LETTERS))
|
|
|
|
|
|
#: Every ISO 3166-1 country by its English name, plus the names boards also
|
|
#: write for it: its own language's for a European one, and the usual short
|
|
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
|
|
_COUNTRY_NAMES = """
|
|
AD Andorra
|
|
AE United Arab Emirates | UAE | U.A.E. | Emirates
|
|
AF Afghanistan
|
|
AG Antigua and Barbuda
|
|
AI Anguilla
|
|
AL Albania | Shqipëria | Shqipëri
|
|
AM Armenia | Hayastan
|
|
AO Angola
|
|
AQ Antarctica
|
|
AR Argentina
|
|
AS American Samoa
|
|
AT Austria | Österreich
|
|
AU Australia
|
|
AW Aruba
|
|
AX Åland Islands | Åland
|
|
AZ Azerbaijan | Azərbaycan
|
|
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
|
|
BB Barbados
|
|
BD Bangladesh
|
|
BE Belgium | België | Belgique | Belgien
|
|
BF Burkina Faso
|
|
BG Bulgaria | България | Balgariya
|
|
BH Bahrain
|
|
BI Burundi
|
|
BJ Benin
|
|
BL Saint Barthélemy
|
|
BM Bermuda
|
|
BN Brunei | Brunei Darussalam
|
|
BO Bolivia
|
|
BQ Caribbean Netherlands | Bonaire
|
|
BR Brazil | Brasil
|
|
BS Bahamas
|
|
BT Bhutan
|
|
BV Bouvet Island
|
|
BW Botswana
|
|
BY Belarus | Беларусь | Belarus'
|
|
BZ Belize
|
|
CA Canada
|
|
CC Cocos Islands | Cocos (Keeling) Islands
|
|
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
|
|
CF Central African Republic
|
|
CG Republic of the Congo | Congo-Brazzaville
|
|
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
|
|
CI Côte d'Ivoire | Ivory Coast
|
|
CK Cook Islands
|
|
CL Chile
|
|
CM Cameroon
|
|
CN China | People's Republic of China
|
|
CO Colombia
|
|
CR Costa Rica
|
|
CU Cuba
|
|
CV Cabo Verde | Cape Verde
|
|
CW Curaçao
|
|
CX Christmas Island
|
|
CY Cyprus | Κύπρος | Kypros | Kibris
|
|
CZ Czechia | Czech Republic | Česko | Česká republika
|
|
DE Germany | Deutschland
|
|
DJ Djibouti
|
|
DK Denmark | Danmark
|
|
DM Dominica
|
|
DO Dominican Republic
|
|
DZ Algeria
|
|
EC Ecuador
|
|
EE Estonia | Eesti
|
|
EG Egypt
|
|
EH Western Sahara
|
|
ER Eritrea
|
|
ES Spain | España | Espanya
|
|
ET Ethiopia
|
|
FI Finland | Suomi
|
|
FJ Fiji
|
|
FK Falkland Islands
|
|
FM Micronesia
|
|
FO Faroe Islands | Føroyar | Færøerne
|
|
FR France
|
|
GA Gabon
|
|
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
|
|
GD Grenada
|
|
GE Georgia | Sakartvelo
|
|
GF French Guiana
|
|
GG Guernsey
|
|
GH Ghana
|
|
GI Gibraltar
|
|
GL Greenland
|
|
GM Gambia
|
|
GN Guinea
|
|
GP Guadeloupe
|
|
GQ Equatorial Guinea
|
|
GR Greece | Ελλάδα | Hellas | Ellada
|
|
GS South Georgia and the South Sandwich Islands
|
|
GT Guatemala
|
|
GU Guam
|
|
GW Guinea-Bissau
|
|
GY Guyana
|
|
HK Hong Kong
|
|
HM Heard Island and McDonald Islands
|
|
HN Honduras
|
|
HR Croatia | Hrvatska
|
|
HT Haiti
|
|
HU Hungary | Magyarország
|
|
ID Indonesia
|
|
IE Ireland | Éire
|
|
IL Israel
|
|
IM Isle of Man
|
|
IN India
|
|
IO British Indian Ocean Territory
|
|
IQ Iraq
|
|
IR Iran
|
|
IS Iceland | Ísland
|
|
IT Italy | Italia
|
|
JE Jersey
|
|
JM Jamaica
|
|
JO Jordan
|
|
JP Japan
|
|
KE Kenya
|
|
KG Kyrgyzstan
|
|
KH Cambodia
|
|
KI Kiribati
|
|
KM Comoros
|
|
KN Saint Kitts and Nevis
|
|
KP North Korea
|
|
KR South Korea | Korea | Republic of Korea
|
|
KW Kuwait
|
|
KY Cayman Islands
|
|
KZ Kazakhstan
|
|
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
|
|
LB Lebanon
|
|
LC Saint Lucia
|
|
LI Liechtenstein
|
|
LK Sri Lanka
|
|
LR Liberia
|
|
LS Lesotho
|
|
LT Lithuania | Lietuva
|
|
LU Luxembourg | Lëtzebuerg | Luxemburg
|
|
LV Latvia | Latvija
|
|
LY Libya
|
|
MA Morocco
|
|
MC Monaco
|
|
MD Moldova | Republic of Moldova
|
|
ME Montenegro | Crna Gora
|
|
MF Saint Martin
|
|
MG Madagascar
|
|
MH Marshall Islands
|
|
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
|
|
ML Mali
|
|
MM Myanmar | Burma
|
|
MN Mongolia
|
|
MO Macao | Macau
|
|
MP Northern Mariana Islands
|
|
MQ Martinique
|
|
MR Mauritania
|
|
MS Montserrat
|
|
MT Malta
|
|
MU Mauritius
|
|
MV Maldives
|
|
MW Malawi
|
|
MX Mexico | México
|
|
MY Malaysia
|
|
MZ Mozambique
|
|
NA Namibia
|
|
NC New Caledonia
|
|
NE Niger
|
|
NF Norfolk Island
|
|
NG Nigeria
|
|
NI Nicaragua
|
|
NL Netherlands | The Netherlands | Nederland | Holland
|
|
NO Norway | Norge | Noreg
|
|
NP Nepal
|
|
NR Nauru
|
|
NU Niue
|
|
NZ New Zealand | Aotearoa
|
|
OM Oman
|
|
PA Panama | Panamá
|
|
PE Peru | Perú
|
|
PF French Polynesia
|
|
PG Papua New Guinea
|
|
PH Philippines
|
|
PK Pakistan
|
|
PL Poland | Polska
|
|
PM Saint Pierre and Miquelon
|
|
PN Pitcairn Islands
|
|
PR Puerto Rico
|
|
PS Palestine
|
|
PT Portugal
|
|
PW Palau
|
|
PY Paraguay
|
|
QA Qatar
|
|
RE Réunion
|
|
RO Romania | România
|
|
RS Serbia | Srbija | Србија
|
|
RU Russia | Russian Federation | Россия | Rossiya
|
|
RW Rwanda
|
|
SA Saudi Arabia | KSA
|
|
SB Solomon Islands
|
|
SC Seychelles
|
|
SD Sudan
|
|
SE Sweden | Sverige
|
|
SG Singapore
|
|
SH Saint Helena
|
|
SI Slovenia | Slovenija
|
|
SJ Svalbard and Jan Mayen | Svalbard
|
|
SK Slovakia | Slovensko
|
|
SL Sierra Leone
|
|
SM San Marino
|
|
SN Senegal
|
|
SO Somalia
|
|
SR Suriname
|
|
SS South Sudan
|
|
ST São Tomé and Príncipe
|
|
SV El Salvador
|
|
SX Sint Maarten
|
|
SY Syria
|
|
SZ Eswatini | Swaziland
|
|
TC Turks and Caicos Islands
|
|
TD Chad
|
|
TF French Southern Territories
|
|
TG Togo
|
|
TH Thailand
|
|
TJ Tajikistan
|
|
TK Tokelau
|
|
TL Timor-Leste | East Timor
|
|
TM Turkmenistan
|
|
TN Tunisia
|
|
TO Tonga
|
|
TR Turkey | Türkiye
|
|
TT Trinidad and Tobago
|
|
TV Tuvalu
|
|
TW Taiwan
|
|
TZ Tanzania
|
|
UA Ukraine | Україна | Ukraina
|
|
UG Uganda
|
|
UM United States Minor Outlying Islands
|
|
US United States | United States of America | USA | U.S.A. | US | U.S.
|
|
UY Uruguay
|
|
UZ Uzbekistan
|
|
VA Vatican City | Holy See
|
|
VC Saint Vincent and the Grenadines
|
|
VE Venezuela
|
|
VG British Virgin Islands
|
|
VI U.S. Virgin Islands | US Virgin Islands
|
|
VN Vietnam | Viet Nam
|
|
VU Vanuatu
|
|
WF Wallis and Futuna
|
|
WS Samoa
|
|
XK Kosovo | Kosova
|
|
YE Yemen
|
|
YT Mayotte
|
|
ZA South Africa
|
|
ZM Zambia
|
|
ZW Zimbabwe
|
|
"""
|
|
|
|
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
|
|
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
|
|
|
|
#: The countries `EU` stands for when a line names several: the UN
|
|
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
|
|
#: the Council of Europe's members in the Caucasus and Turkey, which
|
|
#: European employers list beside their other offices.
|
|
_EUROPE = frozenset((
|
|
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
|
|
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
|
|
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
|
|
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
|
|
"XK",
|
|
))
|
|
_EUROPE_OR_EU = _EUROPE | {"EU"}
|
|
|
|
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
|
|
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
|
|
|
#: The cities boards name most, and a few regions below a country. A US
|
|
#: state or a Canadian province in the same part of the line beats one of
|
|
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
|
_CITIES: dict[str, str] = {
|
|
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
|
|
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
|
|
**dict.fromkeys(("sofia",), "BG"),
|
|
**dict.fromkeys(("zagreb",), "HR"),
|
|
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
|
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
|
|
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
|
|
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
|
|
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
|
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
|
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
|
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
|
|
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
|
"lille", "rennes", "amiens"), "FR"),
|
|
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
|
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
|
|
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
|
|
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
|
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
|
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
|
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
|
|
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
|
**dict.fromkeys(("budapest",), "HU"),
|
|
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
|
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
|
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
|
|
**dict.fromkeys(("riga",), "LV"),
|
|
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
|
**dict.fromkeys(("valletta",), "MT"),
|
|
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
|
"delft"), "NL"),
|
|
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
|
"PL"),
|
|
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
|
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
|
|
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
|
**dict.fromkeys(("bratislava",), "SK"),
|
|
**dict.fromkeys(("ljubljana",), "SI"),
|
|
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
|
"granada", "santiago de compostela"), "ES"),
|
|
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
|
|
**dict.fromkeys(("reykjavik",), "IS"),
|
|
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
|
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
|
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
|
|
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
|
|
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
|
|
"slough", "southampton", "swindon", "liverpool", "redhill",
|
|
"milton keynes"), "GB"),
|
|
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
|
"sankt gallen", "st gallen"), "CH"),
|
|
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
|
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
|
**dict.fromkeys(("tirana",), "AL"),
|
|
**dict.fromkeys(("yerevan",), "AM"),
|
|
**dict.fromkeys(("tbilisi",), "GE"),
|
|
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
|
|
"boston", "chicago", "austin", "los angeles", "palo alto",
|
|
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
|
|
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
|
|
"washington dc", "washington d c", "miami", "dallas", "houston",
|
|
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
|
|
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
|
|
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
|
|
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
|
|
"CA"),
|
|
**dict.fromkeys(("buenos aires",), "AR"),
|
|
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
|
|
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
|
|
**dict.fromkeys(("bogota",), "CO"),
|
|
**dict.fromkeys(("lima", "miraflores"), "PE"),
|
|
**dict.fromkeys(("montevideo",), "UY"),
|
|
**dict.fromkeys(("managua",), "NI"),
|
|
**dict.fromkeys(("santo domingo",), "DO"),
|
|
**dict.fromkeys(("san salvador",), "SV"),
|
|
**dict.fromkeys(("panama city",), "PA"),
|
|
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
|
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
|
|
**dict.fromkeys(("riyadh",), "SA"),
|
|
**dict.fromkeys(("doha",), "QA"),
|
|
**dict.fromkeys(("manama",), "BH"),
|
|
**dict.fromkeys(("salmiya",), "KW"),
|
|
**dict.fromkeys(("muscat",), "OM"),
|
|
**dict.fromkeys(("cairo", "maadi"), "EG"),
|
|
**dict.fromkeys(("casablanca",), "MA"),
|
|
**dict.fromkeys(("dakar",), "SN"),
|
|
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
|
**dict.fromkeys(("almaty",), "KZ"),
|
|
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
|
|
**dict.fromkeys(("dhaka",), "BD"),
|
|
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
|
|
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
|
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
|
|
**dict.fromkeys(("singapore",), "SG"),
|
|
**dict.fromkeys(("manila", "taguig"), "PH"),
|
|
**dict.fromkeys(("bangkok",), "TH"),
|
|
**dict.fromkeys(("phnom penh",), "KH"),
|
|
**dict.fromkeys(("taipei",), "TW"),
|
|
**dict.fromkeys(("tokyo", "osaka"), "JP"),
|
|
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
|
|
**dict.fromkeys(("seoul",), "KR"),
|
|
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
|
|
"queensland"), "AU"),
|
|
}
|
|
|
|
#: US states and Canadian provinces, by name and by the code boards write
|
|
#: after a comma ("Boston, MA", "Toronto, ON").
|
|
_SUBDIVISIONS = """
|
|
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
|
|
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
|
|
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
|
|
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
|
|
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
|
|
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
|
|
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
|
|
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
|
|
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
|
|
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
|
|
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
|
|
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
|
|
CA Nunavut NU
|
|
"""
|
|
|
|
|
|
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
|
|
names: dict[str, str] = {}
|
|
codes: dict[str, str] = {}
|
|
for line in _SUBDIVISIONS.split("\n"):
|
|
if line:
|
|
country, rest = line.split(" ", 1)
|
|
for entry in rest.split("|"):
|
|
name, code = entry.strip().rsplit(" ", 1)
|
|
names[_place_fold(name)] = country
|
|
codes[code] = country
|
|
return names, codes
|
|
|
|
|
|
_STATE_NAMES, _STATE_CODES = _subdivisions()
|
|
|
|
|
|
def _names() -> dict[str, tuple[str, str]]:
|
|
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
|
|
`state`. A name that is a city and a state ("New York") is the city, so
|
|
it does not beat another city in its line. A name that is a country and
|
|
a state ("Georgia") is the country here and read as the state only when
|
|
the line names a place in the US too."""
|
|
names: dict[str, tuple[str, str]] = {}
|
|
for name, code in _STATE_NAMES.items():
|
|
names[name] = (code, "state")
|
|
for name, code in _CITIES.items():
|
|
names[_place_fold(name)] = (code, "city")
|
|
for line in _COUNTRY_NAMES.split("\n"):
|
|
if line:
|
|
code, rest = line.split(" ", 1)
|
|
for name in rest.split("|"):
|
|
names[_place_fold(name)] = (code, "country")
|
|
for name in _REGIONS:
|
|
names[name] = ("EU", "country")
|
|
return names
|
|
|
|
|
|
_NAMES = _names()
|
|
|
|
#: A state name that is also a country's, read as the state only when the
|
|
#: line names a place in that state's country too.
|
|
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
|
|
if _NAMES[name][1] == "country"}
|
|
|
|
#: The longest name in words, so a lookup tries "new york city" before "new
|
|
#: york" and "northern ireland" before "ireland".
|
|
_LONGEST = max(len(name.split()) for name in _NAMES)
|
|
|
|
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
|
|
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
|
|
|
|
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
|
|
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
|
|
|
|
#: Where one location in a line ends and the next begins. A state beats a
|
|
#: city only within one: "London; Ontario" is two places.
|
|
_SEGMENT = re.compile(r"[;|/\n]")
|
|
|
|
|
|
def _hits(folded: str) -> list[tuple[str, str, str]]:
|
|
"""The names in a folded line, longest first at each word, as
|
|
`(name, code, kind)`."""
|
|
words = folded.split()
|
|
hits = []
|
|
i = 0
|
|
while i < len(words):
|
|
for n in range(min(_LONGEST, len(words) - i), 0, -1):
|
|
name = " ".join(words[i:i + n])
|
|
if name in _NAMES:
|
|
hits.append((name, *_NAMES[name]))
|
|
i += n
|
|
break
|
|
else:
|
|
i += 1
|
|
return hits
|
|
|
|
|
|
def country_of(place: str) -> str:
|
|
"""The one country a board's location line names, as an ISO code; `EU`
|
|
when it names several and all of them are in Europe; '' when it names
|
|
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
|
|
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
|
|
New York" and "Remote, EU or US" are ''.
|
|
|
|
Within one location (the line split at `;`, `|` and `/`) the widest unit
|
|
named wins: a country beats a state or a province, which beats a city.
|
|
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
|
|
country also beats the region it is in: "London, EMEA" is GB.
|
|
|
|
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
|
|
count as the state only when the line names a place in the US too:
|
|
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
|
|
than Delaware. A country code in parentheses, "Leicester (GB)", is read
|
|
only when no name matched."""
|
|
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
|
|
named = {code for _, hits in segments for _, code, _ in hits}
|
|
found: set[str] = set()
|
|
for text, hits in segments:
|
|
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
|
|
countries = {code for name, code, kind in hits
|
|
if kind == "country" and code != "EU" and name not in shared}
|
|
states = {code for _, code, kind in hits if kind == "state"}
|
|
states |= {_SHARED_NAMES[name] for name in shared}
|
|
states |= {country for code in _STATE_CODE.findall(text)
|
|
if (country := _STATE_CODES.get(code))
|
|
and (code not in _ISO or country in named)}
|
|
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
|
|
found |= {code for _, code, _ in hits if code == "EU"}
|
|
if not found:
|
|
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
|
|
if len(found) == 1:
|
|
return found.pop()
|
|
if found and found <= _EUROPE_OR_EU:
|
|
countries = found - {"EU"}
|
|
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
|
|
return ""
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
#: Words in a location line that say how the role is worked, for a board that
|
|
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
|
|
#: "Distributed" as the location and nothing else.
|
|
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
|
|
("hybrid", "hybrid"),
|
|
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
|
|
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
|
|
("worldwide", "remote"), ("telecommute", "remote"),
|
|
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
|
|
("in-office", "onsite"), ("in office", "onsite"),
|
|
)
|
|
|
|
|
|
def remote_of(place: str) -> str:
|
|
"""How a board's location line says the role is worked, as one of
|
|
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
|
|
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
|
|
and a line that names only a place is ''."""
|
|
folded = _fold(place or "")
|
|
for word, mode in _REMOTE_WORDS:
|
|
if word in folded:
|
|
return mode
|
|
return ""
|
|
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
|
|
# -------------------------------------------------------------- posting dates
|
|
|
|
#: Hosts whose date is a creation or first-publication timestamp, safe to
|
|
#: read as when a role was posted, checked against live boards on
|
|
#: 2026-09-29:
|
|
#:
|
|
#: - `greenhouse`: `plugins.ats._parse_greenhouse` already prefers
|
|
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
|
#: with `first_published` five months before `updated_at`, proving the
|
|
#: first stays fixed while the second moves on an edit — the resettable
|
|
#: field decisions.md used to warn against is `updated_at`, which this
|
|
#: reads only when `first_published` is missing (rare: 0 of 208 on that
|
|
#: board). The large real gaps this migration measured for Greenhouse
|
|
#: roles (average 143 days between this date and `first_seen`) are further
|
|
#: evidence it holds still rather than tracking every edit.
|
|
#: - `lever` (`createdAt`), `ashby` (`publishedAt`), `personio` (`createdAt`):
|
|
#: named and shaped as creation dates, with no "last edited" field in the
|
|
#: feed at all to reset from, and the same large real gaps as Greenhouse's.
|
|
#: - `smartrecruiters` (`releasedDate`), `recruitee` (`published_at`, falling
|
|
#: back to `created_at`): `releasedDate` reads the same way; a live
|
|
#: Recruitee board (bunq) showed `published_at` and `created_at` minutes
|
|
#: apart while `updated_at` had moved days later on an edited posting,
|
|
#: confirming the field this migration reads is not the one that moves.
|
|
#: - `workable` (`published`): named as a publish date, not an edit one, but
|
|
#: only one live board had postings to check (huggingface, 8 roles) and
|
|
#: none showed a second, "last touched" field to compare against, so this
|
|
#: is trusted on the name and shape alone.
|
|
#: - `jobindex`: the RSS `pubDate` a saved search returns. The live
|
|
#: database's 158 Jobindex-found roles have a `published_at`/`first_seen`
|
|
#: gap of 0.4 to 31 days (median 12), which reads as real, spread-out
|
|
#: posting ages rather than a date that jumps to "now" on every re-crawl.
|
|
#: - `adzuna`: `created`, an aggregator's own record-creation stamp — named
|
|
#: as creation, not modification, like the rest. No live check: this
|
|
#: deployment carries no Adzuna key (`plugins.search`'s own docstring:
|
|
#: "not yet run live") and no Adzuna subscription exists to fetch.
|
|
#:
|
|
#: A host not listed here, or a role whose stored `published_at` fails the
|
|
#: sanity check in `posted_at`, keeps `first_seen` as its posted date — the
|
|
#: safe default this migration is replacing everywhere else.
|
|
TRUSTED_POSTED_HOSTS: frozenset[str] = frozenset({
|
|
"greenhouse", "lever", "ashby", "workable", "smartrecruiters", "recruitee",
|
|
"personio", "jobindex", "adzuna",
|
|
})
|
|
|
|
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
|
#: real posting date: no board here existed before 2015.
|
|
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
|
|
|
|
|
def posted_host(raw: dict[str, Any]) -> str:
|
|
"""The host a role's dates came from, read the way `ats` and `search`
|
|
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
|
`lever`, ...), `provider` for a cross-company search (`jobindex`,
|
|
`adzuna`). '' when neither plugin wrote one."""
|
|
return str(raw.get("board") or raw.get("provider") or "")
|
|
|
|
|
|
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
|
"""The date a role counts as posted: the earlier of `first_seen` (when
|
|
the app itself first stored it) and `published_at` (what the board
|
|
claims), but only when `host` is in `TRUSTED_POSTED_HOSTS` and
|
|
`published_at` is neither absurdly old nor later than `first_seen` — a
|
|
role cannot be posted after the app already saw it, trusted host or not,
|
|
which also catches a `published_at` the board reports in the future
|
|
(`first_seen` is never later than the moment this runs). Both dates are
|
|
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
|
is exact and needs no parsing. A role with no trusted date, or with one
|
|
that fails the check, keeps `first_seen`."""
|
|
if host in TRUSTED_POSTED_HOSTS and _POSTED_FLOOR <= published_at <= first_seen:
|
|
return published_at
|
|
return first_seen
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
seniority: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
|
languages: JsonStrings = []
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
original_url: str = ""
|
|
published_at: str
|
|
#: The date this role counts as posted: see `posted_at`. Used for
|
|
#: `recency` and the feed's newest order; `first_seen` stays what it
|
|
#: always was, when the app itself first stored the role.
|
|
posted_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
#: The vote of the agent the role was read for (see `role_votes`): None
|
|
#: unjudged, else -1 pass, 0 neutral, 1 like.
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
#: NULL unread, else when the feed opened it or judged it. A vote stamps
|
|
#: this too, see `Store.set_vote`: voting is reading.
|
|
read_at: str | None = None
|
|
summary: str = ""
|
|
summarised_at: str | None = None
|
|
#: The posting page as plain text, for a role whose description is a teaser.
|
|
article_text: str = ""
|
|
article_at: str | None = None
|
|
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
|
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
|
tags: list[str] = []
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See app/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class ScoredRole(Role):
|
|
"""A role as `/api/roles` serves it: with one agent's score and gate
|
|
reason when an agent was asked for, and without otherwise."""
|
|
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
|
|
|
|
class Company(Model):
|
|
"""A row of `companies`, with `affinity` as the agent it was read for
|
|
has it (see `agent_companies`); 0 when no agent was asked for."""
|
|
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
#: The host the careers site runs on, readable or not; "" when unknown.
|
|
#: A company with an `ats` and no board subscription is one the app
|
|
#: cannot read yet. See `app.jobs.discover`.
|
|
ats: str = ""
|
|
country: str = ""
|
|
city: str = ""
|
|
#: The name `ats` knows the company by, the `acme` of acme.teamtailor.com;
|
|
#: "" where discovery found none. Kept for a host no parser reads yet.
|
|
ats_slug: str = ""
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
# ------------------------------------------------------------------ agents
|
|
|
|
#: The role columns a gate can name. `company` matches on the company key;
|
|
#: `place` is a case-insensitive substring, like the place filter; the rest are
|
|
#: case-insensitive equality, except `salary_min` (at least the value) and
|
|
#: `languages` (every language the role requires is one of the values).
|
|
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
|
"currency", "permit", "languages", "company"]
|
|
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
|
|
|
GateMode = Literal["require", "any", "prefer"]
|
|
|
|
|
|
class Gate(Model):
|
|
"""One condition of an agent. `require` gates a role that fails it (the
|
|
row is kept with the reason); the `any` gates together are one group a
|
|
role must match at least one of ("in Denmark, or remote"); `prefer` adds
|
|
`weight` to its score when it matches. A gate rejects on evidence only: a
|
|
role whose column is empty is not rejected, since `extract` fills the
|
|
column later for the roles that rank, and only for those."""
|
|
|
|
field: GateField
|
|
values: list[str | int | float] = Field(min_length=1)
|
|
mode: GateMode = "require"
|
|
weight: float = Field(1.0, ge=0)
|
|
|
|
|
|
class Weights(Model):
|
|
"""How much each score term counts. Summed with the prefer gates' weights
|
|
into the denominator, so a score stays in 0..1 whatever the numbers."""
|
|
|
|
fit: float = Field(0.6, ge=0)
|
|
recency: float = Field(0.2, ge=0)
|
|
affinity: float = Field(0.2, ge=0)
|
|
#: Cosine to the liked-minus-passed direction of the agent's votes.
|
|
taste: float = Field(0.2, ge=0)
|
|
|
|
|
|
class Limits(Model):
|
|
"""How far down an agent's ranking the two model stages read. Each pair
|
|
is a top N, the backstop, and a minimum score, the real stop: a role
|
|
is read when it is among the agent's best `extract_top` ungated scores
|
|
and not below `extract_min`, and summarised likewise. A fresh role with
|
|
the worst fit scores about 0.27 on the default weights, a good one 0.6
|
|
or more, so extract's floor is loose and summarise's, the long answer,
|
|
stricter."""
|
|
|
|
extract_top: int = Field(500, ge=0)
|
|
extract_min: float = Field(0.3, ge=0, le=1)
|
|
summarise_top: int = Field(30, ge=0)
|
|
summarise_min: float = Field(0.5, ge=0, le=1)
|
|
|
|
|
|
class Agent(Model):
|
|
"""A row of `agents`: a named search with its own intent, gates, weights
|
|
and limits. The cached intent vector is a column this does not declare."""
|
|
|
|
id: int
|
|
name: str
|
|
intent: str = ""
|
|
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
|
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
|
|
enabled: bool = True
|
|
created_at: str
|
|
|
|
|
|
class Candidate(Model):
|
|
"""What scoring reads about one open role for one agent: its posted
|
|
date (for `recency`), its company's affinity for that agent, and per
|
|
gate `1` match, `0` no match, `-1` the column is empty."""
|
|
|
|
uid: str
|
|
posted_at: str
|
|
affinity: int = 0
|
|
gates: list[int] = []
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|