diff --git a/app/models.py b/app/models.py index 46606d0..9a6b5db 100644 --- a/app/models.py +++ b/app/models.py @@ -66,121 +66,538 @@ def company_key(company: str) -> str: # ---------------------------------------------------------------- geography -#: Folded names a board writes in its location line, to an ISO code: every -#: European country in English and its own language, the cities the boards -#: name most, and the non-European places that fill the seed boards, so those -#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. -_PLACES: dict[str, str] = { - **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), - **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), - **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", - "ghent"), "BE"), - **dict.fromkeys(("bulgaria", "sofia"), "BG"), - **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), - **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), - **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), - # Folding keeps ø and æ, which are letters of their own rather than an o - # or an a with a mark, so København is listed beside Kobenhavn. - **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", - "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", - "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", - "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", - "birkerød"), "DK"), - **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), - **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), - **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", - "bordeaux", "lille"), "FR"), - **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", - "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", - "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", - "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), - **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), - **dict.fromkeys(("hungary", "budapest"), "HU"), - **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), - **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", - "torino", "bologna", "florence"), "IT"), - **dict.fromkeys(("latvia", "riga"), "LV"), - **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), - **dict.fromkeys(("luxembourg",), "LU"), - **dict.fromkeys(("malta", "valletta"), "MT"), - **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", - "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", +#: Letters NFKD leaves whole because they are letters of their own rather +#: than a base letter with a mark, spelt the way boards write them in ASCII. +_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe", + "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"}) + + +def _place_fold(text: str) -> str: + """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København" + meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is, + since it also builds `company_key`, the primary key of `companies`.""" + return _fold((text or "").casefold().translate(_LETTERS)) + + +#: Every ISO 3166-1 country by its English name, plus the names boards also +#: write for it: its own language's for a European one, and the usual short +#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one. +_COUNTRY_NAMES = """ +AD Andorra +AE United Arab Emirates | UAE | U.A.E. | Emirates +AF Afghanistan +AG Antigua and Barbuda +AI Anguilla +AL Albania | Shqipëria | Shqipëri +AM Armenia | Hayastan +AO Angola +AQ Antarctica +AR Argentina +AS American Samoa +AT Austria | Österreich +AU Australia +AW Aruba +AX Åland Islands | Åland +AZ Azerbaijan | Azərbaycan +BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina +BB Barbados +BD Bangladesh +BE Belgium | België | Belgique | Belgien +BF Burkina Faso +BG Bulgaria | България | Balgariya +BH Bahrain +BI Burundi +BJ Benin +BL Saint Barthélemy +BM Bermuda +BN Brunei | Brunei Darussalam +BO Bolivia +BQ Caribbean Netherlands | Bonaire +BR Brazil | Brasil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus | Беларусь | Belarus' +BZ Belize +CA Canada +CC Cocos Islands | Cocos (Keeling) Islands +CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa +CF Central African Republic +CG Republic of the Congo | Congo-Brazzaville +CH Switzerland | Schweiz | Suisse | Svizzera | Svizra +CI Côte d'Ivoire | Ivory Coast +CK Cook Islands +CL Chile +CM Cameroon +CN China | People's Republic of China +CO Colombia +CR Costa Rica +CU Cuba +CV Cabo Verde | Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus | Κύπρος | Kypros | Kibris +CZ Czechia | Czech Republic | Česko | Česká republika +DE Germany | Deutschland +DJ Djibouti +DK Denmark | Danmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia | Eesti +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain | España | Espanya +ET Ethiopia +FI Finland | Suomi +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands | Føroyar | Færøerne +FR France +GA Gabon +GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland +GD Grenada +GE Georgia | Sakartvelo +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece | Ελλάδα | Hellas | Ellada +GS South Georgia and the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island and McDonald Islands +HN Honduras +HR Croatia | Hrvatska +HT Haiti +HU Hungary | Magyarország +ID Indonesia +IE Ireland | Éire +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland | Ísland +IT Italy | Italia +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN Saint Kitts and Nevis +KP North Korea +KR South Korea | Korea | Republic of Korea +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic +LB Lebanon +LC Saint Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania | Lietuva +LU Luxembourg | Lëtzebuerg | Luxemburg +LV Latvia | Latvija +LY Libya +MA Morocco +MC Monaco +MD Moldova | Republic of Moldova +ME Montenegro | Crna Gora +MF Saint Martin +MG Madagascar +MH Marshall Islands +MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија +ML Mali +MM Myanmar | Burma +MN Mongolia +MO Macao | Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico | México +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands | The Netherlands | Nederland | Holland +NO Norway | Norge | Noreg +NP Nepal +NR Nauru +NU Niue +NZ New Zealand | Aotearoa +OM Oman +PA Panama | Panamá +PE Peru | Perú +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland | Polska +PM Saint Pierre and Miquelon +PN Pitcairn Islands +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania | România +RS Serbia | Srbija | Србија +RU Russia | Russian Federation | Россия | Rossiya +RW Rwanda +SA Saudi Arabia | KSA +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden | Sverige +SG Singapore +SH Saint Helena +SI Slovenia | Slovenija +SJ Svalbard and Jan Mayen | Svalbard +SK Slovakia | Slovensko +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST São Tomé and Príncipe +SV El Salvador +SX Sint Maarten +SY Syria +SZ Eswatini | Swaziland +TC Turks and Caicos Islands +TD Chad +TF French Southern Territories +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL Timor-Leste | East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey | Türkiye +TT Trinidad and Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine | Україна | Ukraina +UG Uganda +UM United States Minor Outlying Islands +US United States | United States of America | USA | U.S.A. | US | U.S. +UY Uruguay +UZ Uzbekistan +VA Vatican City | Holy See +VC Saint Vincent and the Grenadines +VE Venezuela +VG British Virgin Islands +VI U.S. Virgin Islands | US Virgin Islands +VN Vietnam | Viet Nam +VU Vanuatu +WF Wallis and Futuna +WS Samoa +XK Kosovo | Kosova +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe +""" + +#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK. +_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line) + +#: The countries `EU` stands for when a line names several: the UN +#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and +#: the Council of Europe's members in the Caucasus and Turkey, which +#: European employers list beside their other offices. +_EUROPE = frozenset(( + "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE", + "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE", + "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL", + "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA", + "XK", +)) +_EUROPE_OR_EU = _EUROPE | {"EU"} + +#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them. +_REGIONS = ("europe", "emea", "eu", "european union", "eea") + +#: The cities boards name most, and a few regions below a country. A US +#: state or a Canadian province in the same part of the line beats one of +#: these ("London, Ontario" is Canada); a country name does not lose to it. +_CITIES: dict[str, str] = { + **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"), + **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"), + **dict.fromkeys(("sofia",), "BG"), + **dict.fromkeys(("zagreb",), "HR"), + **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), + **dict.fromkeys(("prague", "praha", "brno"), "CZ"), + **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", + "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", + "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", + "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), + **dict.fromkeys(("tallinn", "tartu"), "EE"), + **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", + "lille", "rennes", "amiens"), "FR"), + **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", + "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", + "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", + "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", + "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", + "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", + "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"), + **dict.fromkeys(("athens", "thessaloniki"), "GR"), + **dict.fromkeys(("budapest",), "HU"), + **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), + **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", + "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"), + **dict.fromkeys(("riga",), "LV"), + **dict.fromkeys(("vilnius", "kaunas"), "LT"), + **dict.fromkeys(("valletta",), "MT"), + **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), - **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", - "gdansk", "poznan", "lodz"), "PL"), - **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), - "RO"), - **dict.fromkeys(("slovakia", "bratislava"), "SK"), - **dict.fromkeys(("slovenia", "ljubljana"), "SI"), - **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", - "malaga", "bilbao"), "ES"), - **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", - "uppsala"), "SE"), - **dict.fromkeys(("iceland", "reykjavik"), "IS"), - **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), - **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", - "wales", "northern ireland", "london", "manchester", "edinburgh", - "glasgow", "bristol", "cambridge uk", "oxford", "leeds", - "birmingham uk", "belfast"), "GB"), - **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", - "basel", "lausanne", "bern", "zug"), "CH"), - **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", - "nyc", "san francisco", "seattle", "boston", "chicago", "austin", - "los angeles", "palo alto", "mountain view", "san mateo", - "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", - "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", - "houston", "phoenix", "tempe", "portland", "salt lake city", - "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", - "nashville", "minneapolis"), "US"), - **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", - "waterloo"), "CA"), - **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", - "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), + "PL"), + **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), + **dict.fromkeys(("belgrade", "beograd"), "RS"), + **dict.fromkeys(("bratislava",), "SK"), + **dict.fromkeys(("ljubljana",), "SI"), + **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", + "granada", "santiago de compostela"), "ES"), + **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), + **dict.fromkeys(("reykjavik",), "IS"), + **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), + **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", + "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", + "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", + "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", + "slough", "southampton", "swindon", "liverpool", "redhill", + "milton keynes"), "GB"), + **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", + "sankt gallen", "st gallen"), "CH"), + **dict.fromkeys(("kyiv", "kiev"), "UA"), + **dict.fromkeys(("istanbul", "ankara"), "TR"), + **dict.fromkeys(("tirana",), "AL"), + **dict.fromkeys(("yerevan",), "AM"), + **dict.fromkeys(("tbilisi",), "GE"), + **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", + "boston", "chicago", "austin", "los angeles", "palo alto", + "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", + "menlo park", "redwood city", "costa mesa", "denver", "atlanta", + "washington dc", "washington d c", "miami", "dallas", "houston", + "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", + "philadelphia", "detroit", "san diego", "raleigh", "nashville", + "minneapolis", "cambridge ma", "bay area", "arlington"), "US"), + **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), + "CA"), + **dict.fromkeys(("buenos aires",), "AR"), + **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"), + **dict.fromkeys(("santiago", "santiago de chile"), "CL"), + **dict.fromkeys(("bogota",), "CO"), + **dict.fromkeys(("lima", "miraflores"), "PE"), + **dict.fromkeys(("montevideo",), "UY"), + **dict.fromkeys(("managua",), "NI"), + **dict.fromkeys(("santo domingo",), "DO"), + **dict.fromkeys(("san salvador",), "SV"), + **dict.fromkeys(("panama city",), "PA"), + **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("dubai", "abu dhabi"), "AE"), + **dict.fromkeys(("riyadh",), "SA"), + **dict.fromkeys(("doha",), "QA"), + **dict.fromkeys(("manama",), "BH"), + **dict.fromkeys(("salmiya",), "KW"), + **dict.fromkeys(("muscat",), "OM"), + **dict.fromkeys(("cairo", "maadi"), "EG"), + **dict.fromkeys(("casablanca",), "MA"), + **dict.fromkeys(("dakar",), "SN"), + **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("almaty",), "KZ"), + **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), + **dict.fromkeys(("dhaka",), "BD"), + **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", + "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"), **dict.fromkeys(("singapore",), "SG"), - **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), - **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), - **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), - **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), - **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), - **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), - **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), + **dict.fromkeys(("manila", "taguig"), "PH"), + **dict.fromkeys(("bangkok",), "TH"), + **dict.fromkeys(("phnom penh",), "KH"), + **dict.fromkeys(("taipei",), "TW"), + **dict.fromkeys(("tokyo", "osaka"), "JP"), + **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"), + **dict.fromkeys(("seoul",), "KR"), + **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", + "queensland"), "AU"), } -#: Keyed the way a place is folded before it is looked up, so a name can be -#: written as it is spelt. -_PLACES = {_fold(name): code for name, code in _PLACES.items()} +#: US states and Canadian provinces, by name and by the code boards write +#: after a comma ("Boston, MA", "Toronto, ON"). +_SUBDIVISIONS = """ +US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO +US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA +US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY +US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN +US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH +US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND +US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI +US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT +US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY +CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB +CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL +CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT +CA Nunavut NU +""" -#: The longest name first, so "new york city" wins over "york" and "northern -#: ireland" over "ireland". -_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) -#: US state codes after a comma, the way US boards write a city: "Boston, MA". -#: Read only when no name matched, since "DE" and "IN" are countries too. -_US_STATE = re.compile( - r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" - r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" - r"\b") +def _subdivisions() -> tuple[dict[str, str], dict[str, str]]: + names: dict[str, str] = {} + codes: dict[str, str] = {} + for line in _SUBDIVISIONS.split("\n"): + if line: + country, rest = line.split(" ", 1) + for entry in rest.split("|"): + name, code = entry.strip().rsplit(" ", 1) + names[_place_fold(name)] = country + codes[code] = country + return names, codes + + +_STATE_NAMES, _STATE_CODES = _subdivisions() + + +def _names() -> dict[str, tuple[str, str]]: + """Every folded name to `(code, kind)`; kind is `country`, `city` or + `state`. A name that is a city and a state ("New York") is the city, so + it does not beat another city in its line. A name that is a country and + a state ("Georgia") is the country here and read as the state only when + the line names a place in the US too.""" + names: dict[str, tuple[str, str]] = {} + for name, code in _STATE_NAMES.items(): + names[name] = (code, "state") + for name, code in _CITIES.items(): + names[_place_fold(name)] = (code, "city") + for line in _COUNTRY_NAMES.split("\n"): + if line: + code, rest = line.split(" ", 1) + for name in rest.split("|"): + names[_place_fold(name)] = (code, "country") + for name in _REGIONS: + names[name] = ("EU", "country") + return names + + +_NAMES = _names() + +#: A state name that is also a country's, read as the state only when the +#: line names a place in that state's country too. +_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES + if _NAMES[name][1] == "country"} + +#: The longest name in words, so a lookup tries "new york city" before "new +#: york" and "northern ireland" before "ireland". +_LONGEST = max(len(name.split()) for name in _NAMES) + +#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin". +_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b") + +#: A country code in parentheses, "Leicester (GB)", read only when no name matched. +_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)") + +#: Where one location in a line ends and the next begins. A state beats a +#: city only within one: "London; Ontario" is two places. +_SEGMENT = re.compile(r"[;|/\n]") + + +def _hits(folded: str) -> list[tuple[str, str, str]]: + """The names in a folded line, longest first at each word, as + `(name, code, kind)`.""" + words = folded.split() + hits = [] + i = 0 + while i < len(words): + for n in range(min(_LONGEST, len(words) - i), 0, -1): + name = " ".join(words[i:i + n]) + if name in _NAMES: + hits.append((name, *_NAMES[name])) + i += n + break + else: + i += 1 + return hits def country_of(place: str) -> str: - """The one country a board's location line names, as an ISO code, or '' - when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is - US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" - is EU, and "London, EMEA" is GB: a country beats the region it is in.""" - folded = f" {_fold(place or '')} " + """The one country a board's location line names, as an ISO code; `EU` + when it names several and all of them are in Europe; '' when it names + none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston, + MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or + New York" and "Remote, EU or US" are ''. + + Within one location (the line split at `;`, `|` and `/`) the widest unit + named wins: a country beats a state or a province, which beats a city. + So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A + country also beats the region it is in: "London, EMEA" is GB. + + A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", + count as the state only when the line names a place in the US too: + "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather + than Delaware. A country code in parentheses, "Leicester (GB)", is read + only when no name matched.""" + segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] + named = {code for _, hits in segments for _, code, _ in hits} found: set[str] = set() - for name in _PLACE_NAMES: - needle = f" {name} " - if needle in folded: - found.add(_PLACES[name]) - folded = folded.replace(needle, " ") - if not found and _US_STATE.search(place or ""): - # Only when nothing else named a country: "Berlin, DE" is Germany. - found.add("US") - if len(found) > 1: - found.discard("EU") - return found.pop() if len(found) == 1 else "" + for text, hits in segments: + shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named} + countries = {code for name, code, kind in hits + if kind == "country" and code != "EU" and name not in shared} + states = {code for _, code, kind in hits if kind == "state"} + states |= {_SHARED_NAMES[name] for name in shared} + states |= {country for code in _STATE_CODE.findall(text) + if (country := _STATE_CODES.get(code)) + and (code not in _ISO or country in named)} + found |= (countries or states or {code for _, code, _ in hits if code != "EU"}) + found |= {code for _, code, _ in hits if code == "EU"} + if not found: + found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO} + if len(found) == 1: + return found.pop() + if found and found <= _EUROPE_OR_EU: + countries = found - {"EU"} + return countries.pop() if "EU" in found and len(countries) == 1 else "EU" + return "" # --------------------------------------------------------------- vocabulary diff --git a/app/storage/schema.py b/app/storage/schema.py index 3219e87..5f6b2e7 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -17,7 +17,7 @@ import logging import sqlite3 from collections.abc import Callable -from app.models import posted_at, posted_host +from app.models import country_of, posted_at, posted_host log = logging.getLogger(__name__) @@ -381,6 +381,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; # its current rules is the whole point (see _recompute_function below). (11, "roles.function recomputed for every not-yet-extracted role, word-boundary " "classifier fix", lambda conn: _recompute_function(conn)), + (12, "roles.country recomputed from place for every not-yet-extracted role whose " + "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -429,6 +431,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None: len(updates), len(rows)) +def _recompute_country(conn: sqlite3.Connection) -> None: + """Migration 12's step. `app.models.country_of` used to read a US state + code over a country's ("Bogota, CO" was US), return '' for a line naming + several European countries, and miss most countries outside Europe; + `Store.backfill_from_place` fills only an empty column, so a wrong code + would stay. Recomputed from `place` for every role `extract` has not + read (its answer only ever filled an empty column, but a role it read is + left as it is) and whose board sent no `country` of its own in `raw` + (the `search` plugin's Jobindex and Adzuna rows do), which is what + `Store.upsert_role` would write for that role today.""" + rows = conn.execute( + "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall() + updates = [] + for uid, place, raw, old in rows: + try: + given = json.loads(raw or "{}").get("country") or "" + except (TypeError, ValueError, AttributeError): + given = "" + if given: + continue + new = country_of(place or "") + if new != old: + updates.append((new, uid)) + if updates: + conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates) + log.info("migration 12: recomputed country for %d of %d not-yet-extracted roles", + len(updates), len(rows)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index d95264f..d018147 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -42,7 +42,7 @@ Advanced + Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: [{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}] is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.
diff --git a/docs/decisions.md b/docs/decisions.md index af07529..41736ae 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -32,7 +32,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. -**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. +**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the location line to an ISO code from every ISO country by its English name (and its own-language name for a European one) and the cities the boards name most, where `extract` would be a paid call per role for the same answer. On the live data (2026-09-29) 7.0% of open roles resolve to nothing, down from 16.8%, and most of those write only "Hybrid", "Remote" or a region such as "Americas". The line is folded by `_place_fold`, which also maps ł, đ, ø and æ since NFKD leaves them whole; `_fold` stays as it is because it builds `company_key`. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a US state or a Canadian province, which beats a city, so "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A state code that is also a country's (DE, CO, MA, IN) counts as the state only when the line also names a US place, so "Bogota, CO" is CO and "Paderborn, DE" is ''. A line naming several countries is `EU` when all of them are European (the UN's Europe, plus Cyprus, Turkey and the Council of Europe's Caucasus members) and '' otherwise: "Dublin, London" is EU, "Remote, EU or US" is ''. `EU`, ISO's reserved code, also stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it; a country beats the region it is in ("London, EMEA" is GB). `backfill_from_place` only fills an empty column, so a fix that changes answers comes with a migration: v12 recomputes `country` for every role `extract` has not read and whose board sent no `country` in `raw`, which changed 1,705 open roles, 48 of them from one code to another or to '' (25 Hong Kong roles moved from CN to HK). **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. diff --git a/tests/test_search.py b/tests/test_search.py index a18f404..813bff4 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store: ("London or New York", ""), ("Remote", ""), ("", ""), + ("Wrocław", "PL"), # ł and đ do not decompose under NFKD + ("Łódź, Poland", "PL"), + ("Bogota, CO", "CO"), # a state code that is a country's too + ("Paderborn, DE", ""), # ... is no US state without a US place + ("Sliema, MT", ""), + ("Casablanca, MA", "MA"), + ("Cambridge, MA", "US"), + ("Chicago, IL", "US"), + ("Madrid, MD, Spain", "ES"), + ("Dublin, London", "EU"), # several European countries + ("Anywhere in France, Belgium, Spain", "EU"), + ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), + ("Remote, EU or US", ""), # a region and a country outside it + ("London, Ontario", "CA"), # a province beats the city before it + ("Bellevue, Washington", "US"), + ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code + ("Atlanta, Georgia", "US"), + ("Tbilisi, Georgia", "GE"), + ("Malaysia", "MY"), + ("Belgrade, Serbia", "RS"), + ("Dubai", "AE"), + ("United Arab Emirates", "AE"), + ("Buenos Aires, , Argentina", "AR"), + ("Leicester (GB)", "GB"), # a code in parentheses when nothing else + ("Cagliari (CA)", "IT"), # ... and not over a name + ("Hybrid", ""), + ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code diff --git a/tests/test_storage.py b/tests/test_storage.py index 0fb2a8a..0de97c4 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 11 + assert LATEST == 12 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -257,6 +257,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen +def test_v12_recomputes_country_from_place_unless_the_board_or_extract_set_it( + tmp_path: Path, clock: Clock) -> None: + path = tmp_path / "v11.db" + store = Store(path) + store.clock = clock + wrong = posting("1", place="Bogota, CO") # was read as US + several = posting("2", place="Dublin, London") # was '' + given = posting("3", place="Bogota, CO", country="US") # the board said so + extracted = posting("4", place="Bogota, CO") + for item in (wrong, several, given, extracted): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") + conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) + conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", + (extracted.uid,)) + conn.execute("PRAGMA user_version = 11") + + migrated = Store(path) + + assert migrated.migrations_applied == list(range(12, LATEST + 1)) + countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid) + if (role := migrated.role(uid)) is not None} + assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US", + extracted.uid: "US"} + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23"