diff --git a/app/models.py b/app/models.py
index 46606d0..9a6b5db 100644
--- a/app/models.py
+++ b/app/models.py
@@ -66,121 +66,538 @@ def company_key(company: str) -> str:
# ---------------------------------------------------------------- geography
-#: Folded names a board writes in its location line, to an ISO code: every
-#: European country in English and its own language, the cities the boards
-#: name most, and the non-European places that fill the seed boards, so those
-#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
-_PLACES: dict[str, str] = {
- **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
- **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
- **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
- "ghent"), "BE"),
- **dict.fromkeys(("bulgaria", "sofia"), "BG"),
- **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
- **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
- **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
- # Folding keeps ø and æ, which are letters of their own rather than an o
- # or an a with a mark, so København is listed beside Kobenhavn.
- **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
- "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
- "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
- "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
- "birkerød"), "DK"),
- **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
- **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
- **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
- "bordeaux", "lille"), "FR"),
- **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
- "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
- "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
- "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
- **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
- **dict.fromkeys(("hungary", "budapest"), "HU"),
- **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
- **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
- "torino", "bologna", "florence"), "IT"),
- **dict.fromkeys(("latvia", "riga"), "LV"),
- **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
- **dict.fromkeys(("luxembourg",), "LU"),
- **dict.fromkeys(("malta", "valletta"), "MT"),
- **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
- "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
+#: Letters NFKD leaves whole because they are letters of their own rather
+#: than a base letter with a mark, spelt the way boards write them in ASCII.
+_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
+ "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
+
+
+def _place_fold(text: str) -> str:
+ """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
+ meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
+ since it also builds `company_key`, the primary key of `companies`."""
+ return _fold((text or "").casefold().translate(_LETTERS))
+
+
+#: Every ISO 3166-1 country by its English name, plus the names boards also
+#: write for it: its own language's for a European one, and the usual short
+#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
+_COUNTRY_NAMES = """
+AD Andorra
+AE United Arab Emirates | UAE | U.A.E. | Emirates
+AF Afghanistan
+AG Antigua and Barbuda
+AI Anguilla
+AL Albania | Shqipëria | Shqipëri
+AM Armenia | Hayastan
+AO Angola
+AQ Antarctica
+AR Argentina
+AS American Samoa
+AT Austria | Österreich
+AU Australia
+AW Aruba
+AX Åland Islands | Åland
+AZ Azerbaijan | Azərbaycan
+BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
+BB Barbados
+BD Bangladesh
+BE Belgium | België | Belgique | Belgien
+BF Burkina Faso
+BG Bulgaria | България | Balgariya
+BH Bahrain
+BI Burundi
+BJ Benin
+BL Saint Barthélemy
+BM Bermuda
+BN Brunei | Brunei Darussalam
+BO Bolivia
+BQ Caribbean Netherlands | Bonaire
+BR Brazil | Brasil
+BS Bahamas
+BT Bhutan
+BV Bouvet Island
+BW Botswana
+BY Belarus | Беларусь | Belarus'
+BZ Belize
+CA Canada
+CC Cocos Islands | Cocos (Keeling) Islands
+CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
+CF Central African Republic
+CG Republic of the Congo | Congo-Brazzaville
+CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
+CI Côte d'Ivoire | Ivory Coast
+CK Cook Islands
+CL Chile
+CM Cameroon
+CN China | People's Republic of China
+CO Colombia
+CR Costa Rica
+CU Cuba
+CV Cabo Verde | Cape Verde
+CW Curaçao
+CX Christmas Island
+CY Cyprus | Κύπρος | Kypros | Kibris
+CZ Czechia | Czech Republic | Česko | Česká republika
+DE Germany | Deutschland
+DJ Djibouti
+DK Denmark | Danmark
+DM Dominica
+DO Dominican Republic
+DZ Algeria
+EC Ecuador
+EE Estonia | Eesti
+EG Egypt
+EH Western Sahara
+ER Eritrea
+ES Spain | España | Espanya
+ET Ethiopia
+FI Finland | Suomi
+FJ Fiji
+FK Falkland Islands
+FM Micronesia
+FO Faroe Islands | Føroyar | Færøerne
+FR France
+GA Gabon
+GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
+GD Grenada
+GE Georgia | Sakartvelo
+GF French Guiana
+GG Guernsey
+GH Ghana
+GI Gibraltar
+GL Greenland
+GM Gambia
+GN Guinea
+GP Guadeloupe
+GQ Equatorial Guinea
+GR Greece | Ελλάδα | Hellas | Ellada
+GS South Georgia and the South Sandwich Islands
+GT Guatemala
+GU Guam
+GW Guinea-Bissau
+GY Guyana
+HK Hong Kong
+HM Heard Island and McDonald Islands
+HN Honduras
+HR Croatia | Hrvatska
+HT Haiti
+HU Hungary | Magyarország
+ID Indonesia
+IE Ireland | Éire
+IL Israel
+IM Isle of Man
+IN India
+IO British Indian Ocean Territory
+IQ Iraq
+IR Iran
+IS Iceland | Ísland
+IT Italy | Italia
+JE Jersey
+JM Jamaica
+JO Jordan
+JP Japan
+KE Kenya
+KG Kyrgyzstan
+KH Cambodia
+KI Kiribati
+KM Comoros
+KN Saint Kitts and Nevis
+KP North Korea
+KR South Korea | Korea | Republic of Korea
+KW Kuwait
+KY Cayman Islands
+KZ Kazakhstan
+LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
+LB Lebanon
+LC Saint Lucia
+LI Liechtenstein
+LK Sri Lanka
+LR Liberia
+LS Lesotho
+LT Lithuania | Lietuva
+LU Luxembourg | Lëtzebuerg | Luxemburg
+LV Latvia | Latvija
+LY Libya
+MA Morocco
+MC Monaco
+MD Moldova | Republic of Moldova
+ME Montenegro | Crna Gora
+MF Saint Martin
+MG Madagascar
+MH Marshall Islands
+MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
+ML Mali
+MM Myanmar | Burma
+MN Mongolia
+MO Macao | Macau
+MP Northern Mariana Islands
+MQ Martinique
+MR Mauritania
+MS Montserrat
+MT Malta
+MU Mauritius
+MV Maldives
+MW Malawi
+MX Mexico | México
+MY Malaysia
+MZ Mozambique
+NA Namibia
+NC New Caledonia
+NE Niger
+NF Norfolk Island
+NG Nigeria
+NI Nicaragua
+NL Netherlands | The Netherlands | Nederland | Holland
+NO Norway | Norge | Noreg
+NP Nepal
+NR Nauru
+NU Niue
+NZ New Zealand | Aotearoa
+OM Oman
+PA Panama | Panamá
+PE Peru | Perú
+PF French Polynesia
+PG Papua New Guinea
+PH Philippines
+PK Pakistan
+PL Poland | Polska
+PM Saint Pierre and Miquelon
+PN Pitcairn Islands
+PR Puerto Rico
+PS Palestine
+PT Portugal
+PW Palau
+PY Paraguay
+QA Qatar
+RE Réunion
+RO Romania | România
+RS Serbia | Srbija | Србија
+RU Russia | Russian Federation | Россия | Rossiya
+RW Rwanda
+SA Saudi Arabia | KSA
+SB Solomon Islands
+SC Seychelles
+SD Sudan
+SE Sweden | Sverige
+SG Singapore
+SH Saint Helena
+SI Slovenia | Slovenija
+SJ Svalbard and Jan Mayen | Svalbard
+SK Slovakia | Slovensko
+SL Sierra Leone
+SM San Marino
+SN Senegal
+SO Somalia
+SR Suriname
+SS South Sudan
+ST São Tomé and Príncipe
+SV El Salvador
+SX Sint Maarten
+SY Syria
+SZ Eswatini | Swaziland
+TC Turks and Caicos Islands
+TD Chad
+TF French Southern Territories
+TG Togo
+TH Thailand
+TJ Tajikistan
+TK Tokelau
+TL Timor-Leste | East Timor
+TM Turkmenistan
+TN Tunisia
+TO Tonga
+TR Turkey | Türkiye
+TT Trinidad and Tobago
+TV Tuvalu
+TW Taiwan
+TZ Tanzania
+UA Ukraine | Україна | Ukraina
+UG Uganda
+UM United States Minor Outlying Islands
+US United States | United States of America | USA | U.S.A. | US | U.S.
+UY Uruguay
+UZ Uzbekistan
+VA Vatican City | Holy See
+VC Saint Vincent and the Grenadines
+VE Venezuela
+VG British Virgin Islands
+VI U.S. Virgin Islands | US Virgin Islands
+VN Vietnam | Viet Nam
+VU Vanuatu
+WF Wallis and Futuna
+WS Samoa
+XK Kosovo | Kosova
+YE Yemen
+YT Mayotte
+ZA South Africa
+ZM Zambia
+ZW Zimbabwe
+"""
+
+#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
+_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
+
+#: The countries `EU` stands for when a line names several: the UN
+#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
+#: the Council of Europe's members in the Caucasus and Turkey, which
+#: European employers list beside their other offices.
+_EUROPE = frozenset((
+ "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
+ "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
+ "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
+ "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
+ "XK",
+))
+_EUROPE_OR_EU = _EUROPE | {"EU"}
+
+#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
+_REGIONS = ("europe", "emea", "eu", "european union", "eea")
+
+#: The cities boards name most, and a few regions below a country. A US
+#: state or a Canadian province in the same part of the line beats one of
+#: these ("London, Ontario" is Canada); a country name does not lose to it.
+_CITIES: dict[str, str] = {
+ **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
+ **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
+ **dict.fromkeys(("sofia",), "BG"),
+ **dict.fromkeys(("zagreb",), "HR"),
+ **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
+ **dict.fromkeys(("prague", "praha", "brno"), "CZ"),
+ **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
+ "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
+ "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
+ "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
+ **dict.fromkeys(("tallinn", "tartu"), "EE"),
+ **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
+ **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
+ "lille", "rennes", "amiens"), "FR"),
+ **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
+ "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
+ "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
+ "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
+ "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
+ "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
+ "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
+ **dict.fromkeys(("athens", "thessaloniki"), "GR"),
+ **dict.fromkeys(("budapest",), "HU"),
+ **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
+ **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
+ "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
+ **dict.fromkeys(("riga",), "LV"),
+ **dict.fromkeys(("vilnius", "kaunas"), "LT"),
+ **dict.fromkeys(("valletta",), "MT"),
+ **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
- **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
- "gdansk", "poznan", "lodz"), "PL"),
- **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
- **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
- "RO"),
- **dict.fromkeys(("slovakia", "bratislava"), "SK"),
- **dict.fromkeys(("slovenia", "ljubljana"), "SI"),
- **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
- "malaga", "bilbao"), "ES"),
- **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
- "uppsala"), "SE"),
- **dict.fromkeys(("iceland", "reykjavik"), "IS"),
- **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
- **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
- "wales", "northern ireland", "london", "manchester", "edinburgh",
- "glasgow", "bristol", "cambridge uk", "oxford", "leeds",
- "birmingham uk", "belfast"), "GB"),
- **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
- "basel", "lausanne", "bern", "zug"), "CH"),
- **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
- "nyc", "san francisco", "seattle", "boston", "chicago", "austin",
- "los angeles", "palo alto", "mountain view", "san mateo",
- "foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
- "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
- "houston", "phoenix", "tempe", "portland", "salt lake city",
- "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
- "nashville", "minneapolis"), "US"),
- **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
- "waterloo"), "CA"),
- **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
- "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
+ **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
+ "PL"),
+ **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
+ **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
+ **dict.fromkeys(("belgrade", "beograd"), "RS"),
+ **dict.fromkeys(("bratislava",), "SK"),
+ **dict.fromkeys(("ljubljana",), "SI"),
+ **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
+ "granada", "santiago de compostela"), "ES"),
+ **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
+ **dict.fromkeys(("reykjavik",), "IS"),
+ **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
+ **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
+ "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
+ "guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
+ "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
+ "slough", "southampton", "swindon", "liverpool", "redhill",
+ "milton keynes"), "GB"),
+ **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
+ "sankt gallen", "st gallen"), "CH"),
+ **dict.fromkeys(("kyiv", "kiev"), "UA"),
+ **dict.fromkeys(("istanbul", "ankara"), "TR"),
+ **dict.fromkeys(("tirana",), "AL"),
+ **dict.fromkeys(("yerevan",), "AM"),
+ **dict.fromkeys(("tbilisi",), "GE"),
+ **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
+ "boston", "chicago", "austin", "los angeles", "palo alto",
+ "mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
+ "menlo park", "redwood city", "costa mesa", "denver", "atlanta",
+ "washington dc", "washington d c", "miami", "dallas", "houston",
+ "phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
+ "philadelphia", "detroit", "san diego", "raleigh", "nashville",
+ "minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
+ **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
+ "CA"),
+ **dict.fromkeys(("buenos aires",), "AR"),
+ **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
+ **dict.fromkeys(("santiago", "santiago de chile"), "CL"),
+ **dict.fromkeys(("bogota",), "CO"),
+ **dict.fromkeys(("lima", "miraflores"), "PE"),
+ **dict.fromkeys(("montevideo",), "UY"),
+ **dict.fromkeys(("managua",), "NI"),
+ **dict.fromkeys(("santo domingo",), "DO"),
+ **dict.fromkeys(("san salvador",), "SV"),
+ **dict.fromkeys(("panama city",), "PA"),
+ **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
+ **dict.fromkeys(("dubai", "abu dhabi"), "AE"),
+ **dict.fromkeys(("riyadh",), "SA"),
+ **dict.fromkeys(("doha",), "QA"),
+ **dict.fromkeys(("manama",), "BH"),
+ **dict.fromkeys(("salmiya",), "KW"),
+ **dict.fromkeys(("muscat",), "OM"),
+ **dict.fromkeys(("cairo", "maadi"), "EG"),
+ **dict.fromkeys(("casablanca",), "MA"),
+ **dict.fromkeys(("dakar",), "SN"),
+ **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
+ **dict.fromkeys(("almaty",), "KZ"),
+ **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
+ **dict.fromkeys(("dhaka",), "BD"),
+ **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
+ "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
+ **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
**dict.fromkeys(("singapore",), "SG"),
- **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
- **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
- **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
- **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
- **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
- **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
- **dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
+ **dict.fromkeys(("manila", "taguig"), "PH"),
+ **dict.fromkeys(("bangkok",), "TH"),
+ **dict.fromkeys(("phnom penh",), "KH"),
+ **dict.fromkeys(("taipei",), "TW"),
+ **dict.fromkeys(("tokyo", "osaka"), "JP"),
+ **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
+ **dict.fromkeys(("seoul",), "KR"),
+ **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
+ "queensland"), "AU"),
}
-#: Keyed the way a place is folded before it is looked up, so a name can be
-#: written as it is spelt.
-_PLACES = {_fold(name): code for name, code in _PLACES.items()}
+#: US states and Canadian provinces, by name and by the code boards write
+#: after a comma ("Boston, MA", "Toronto, ON").
+_SUBDIVISIONS = """
+US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
+US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
+US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
+US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
+US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
+US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
+US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
+US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
+US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
+CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
+CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
+CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
+CA Nunavut NU
+"""
-#: The longest name first, so "new york city" wins over "york" and "northern
-#: ireland" over "ireland".
-_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
-#: US state codes after a comma, the way US boards write a city: "Boston, MA".
-#: Read only when no name matched, since "DE" and "IN" are countries too.
-_US_STATE = re.compile(
- r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
- r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
- r"\b")
+def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
+ names: dict[str, str] = {}
+ codes: dict[str, str] = {}
+ for line in _SUBDIVISIONS.split("\n"):
+ if line:
+ country, rest = line.split(" ", 1)
+ for entry in rest.split("|"):
+ name, code = entry.strip().rsplit(" ", 1)
+ names[_place_fold(name)] = country
+ codes[code] = country
+ return names, codes
+
+
+_STATE_NAMES, _STATE_CODES = _subdivisions()
+
+
+def _names() -> dict[str, tuple[str, str]]:
+ """Every folded name to `(code, kind)`; kind is `country`, `city` or
+ `state`. A name that is a city and a state ("New York") is the city, so
+ it does not beat another city in its line. A name that is a country and
+ a state ("Georgia") is the country here and read as the state only when
+ the line names a place in the US too."""
+ names: dict[str, tuple[str, str]] = {}
+ for name, code in _STATE_NAMES.items():
+ names[name] = (code, "state")
+ for name, code in _CITIES.items():
+ names[_place_fold(name)] = (code, "city")
+ for line in _COUNTRY_NAMES.split("\n"):
+ if line:
+ code, rest = line.split(" ", 1)
+ for name in rest.split("|"):
+ names[_place_fold(name)] = (code, "country")
+ for name in _REGIONS:
+ names[name] = ("EU", "country")
+ return names
+
+
+_NAMES = _names()
+
+#: A state name that is also a country's, read as the state only when the
+#: line names a place in that state's country too.
+_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
+ if _NAMES[name][1] == "country"}
+
+#: The longest name in words, so a lookup tries "new york city" before "new
+#: york" and "northern ireland" before "ireland".
+_LONGEST = max(len(name.split()) for name in _NAMES)
+
+#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
+_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
+
+#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
+_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
+
+#: Where one location in a line ends and the next begins. A state beats a
+#: city only within one: "London; Ontario" is two places.
+_SEGMENT = re.compile(r"[;|/\n]")
+
+
+def _hits(folded: str) -> list[tuple[str, str, str]]:
+ """The names in a folded line, longest first at each word, as
+ `(name, code, kind)`."""
+ words = folded.split()
+ hits = []
+ i = 0
+ while i < len(words):
+ for n in range(min(_LONGEST, len(words) - i), 0, -1):
+ name = " ".join(words[i:i + n])
+ if name in _NAMES:
+ hits.append((name, *_NAMES[name]))
+ i += n
+ break
+ else:
+ i += 1
+ return hits
def country_of(place: str) -> str:
- """The one country a board's location line names, as an ISO code, or ''
- when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
- US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
- is EU, and "London, EMEA" is GB: a country beats the region it is in."""
- folded = f" {_fold(place or '')} "
+ """The one country a board's location line names, as an ISO code; `EU`
+ when it names several and all of them are in Europe; '' when it names
+ none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
+ MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
+ New York" and "Remote, EU or US" are ''.
+
+ Within one location (the line split at `;`, `|` and `/`) the widest unit
+ named wins: a country beats a state or a province, which beats a city.
+ So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
+ country also beats the region it is in: "London, EMEA" is GB.
+
+ A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
+ count as the state only when the line names a place in the US too:
+ "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
+ than Delaware. A country code in parentheses, "Leicester (GB)", is read
+ only when no name matched."""
+ segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
+ named = {code for _, hits in segments for _, code, _ in hits}
found: set[str] = set()
- for name in _PLACE_NAMES:
- needle = f" {name} "
- if needle in folded:
- found.add(_PLACES[name])
- folded = folded.replace(needle, " ")
- if not found and _US_STATE.search(place or ""):
- # Only when nothing else named a country: "Berlin, DE" is Germany.
- found.add("US")
- if len(found) > 1:
- found.discard("EU")
- return found.pop() if len(found) == 1 else ""
+ for text, hits in segments:
+ shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
+ countries = {code for name, code, kind in hits
+ if kind == "country" and code != "EU" and name not in shared}
+ states = {code for _, code, kind in hits if kind == "state"}
+ states |= {_SHARED_NAMES[name] for name in shared}
+ states |= {country for code in _STATE_CODE.findall(text)
+ if (country := _STATE_CODES.get(code))
+ and (code not in _ISO or country in named)}
+ found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
+ found |= {code for _, code, _ in hits if code == "EU"}
+ if not found:
+ found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
+ if len(found) == 1:
+ return found.pop()
+ if found and found <= _EUROPE_OR_EU:
+ countries = found - {"EU"}
+ return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
+ return ""
# --------------------------------------------------------------- vocabulary
diff --git a/app/storage/schema.py b/app/storage/schema.py
index 3219e87..5f6b2e7 100644
--- a/app/storage/schema.py
+++ b/app/storage/schema.py
@@ -17,7 +17,7 @@ import logging
import sqlite3
from collections.abc import Callable
-from app.models import posted_at, posted_host
+from app.models import country_of, posted_at, posted_host
log = logging.getLogger(__name__)
@@ -381,6 +381,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
# its current rules is the whole point (see _recompute_function below).
(11, "roles.function recomputed for every not-yet-extracted role, word-boundary "
"classifier fix", lambda conn: _recompute_function(conn)),
+ (12, "roles.country recomputed from place for every not-yet-extracted role whose "
+ "board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -429,6 +431,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
+def _recompute_country(conn: sqlite3.Connection) -> None:
+ """Migration 12's step. `app.models.country_of` used to read a US state
+ code over a country's ("Bogota, CO" was US), return '' for a line naming
+ several European countries, and miss most countries outside Europe;
+ `Store.backfill_from_place` fills only an empty column, so a wrong code
+ would stay. Recomputed from `place` for every role `extract` has not
+ read (its answer only ever filled an empty column, but a role it read is
+ left as it is) and whose board sent no `country` of its own in `raw`
+ (the `search` plugin's Jobindex and Adzuna rows do), which is what
+ `Store.upsert_role` would write for that role today."""
+ rows = conn.execute(
+ "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
+ updates = []
+ for uid, place, raw, old in rows:
+ try:
+ given = json.loads(raw or "{}").get("country") or ""
+ except (TypeError, ValueError, AttributeError):
+ given = ""
+ if given:
+ continue
+ new = country_of(place or "")
+ if new != old:
+ updates.append((new, uid))
+ if updates:
+ conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
+ log.info("migration 12: recomputed country for %d of %d not-yet-extracted roles",
+ len(updates), len(rows))
+
+
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html
index d95264f..d018147 100644
--- a/app/templates/settings/agents.html
+++ b/app/templates/settings/agents.html
@@ -42,7 +42,7 @@
[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}] is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.