From 2c4b992c50758980f5b2b930c17848a6aa7095b7 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:06:15 +0200 Subject: [PATCH] Country lookup: every ISO country, the widest unit wins, several European countries are EU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit country_of folds with _place_fold (ł, đ, ø, æ map to ASCII; _fold keeps building company_key unchanged), knows every ISO country by English name and European ones by their own, and the cities among the 150 most frequent unresolved places on the live data. Within one location a country beats a US state or Canadian province, which beats a city. A state code that is also an ISO code counts as the state only when the line names a US place. Several countries, all European, are EU; otherwise ''. A code in parentheses is read when no name matched. Migration 12 recomputes country from place for roles extract has not read and whose board sent no country in raw. On the live data: open roles with no country 2,827 -> 1,183 of 16,841; 1,705 open roles change, 48 from a code. Co-Authored-By: Claude Opus --- app/models.py | 627 ++++++++++++++++++++++++----- app/storage/schema.py | 33 +- app/templates/settings/agents.html | 2 +- docs/decisions.md | 2 +- tests/test_search.py | 27 ++ tests/test_storage.py | 29 +- 6 files changed, 611 insertions(+), 109 deletions(-) diff --git a/app/models.py b/app/models.py index 46606d0..9a6b5db 100644 --- a/app/models.py +++ b/app/models.py @@ -66,121 +66,538 @@ def company_key(company: str) -> str: # ---------------------------------------------------------------- geography -#: Folded names a board writes in its location line, to an ISO code: every -#: European country in English and its own language, the cities the boards -#: name most, and the non-European places that fill the seed boards, so those -#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. -_PLACES: dict[str, str] = { - **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), - **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), - **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", - "ghent"), "BE"), - **dict.fromkeys(("bulgaria", "sofia"), "BG"), - **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), - **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), - **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), - # Folding keeps ø and æ, which are letters of their own rather than an o - # or an a with a mark, so København is listed beside Kobenhavn. - **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", - "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", - "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", - "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", - "birkerød"), "DK"), - **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), - **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), - **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", - "bordeaux", "lille"), "FR"), - **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", - "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", - "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", - "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), - **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), - **dict.fromkeys(("hungary", "budapest"), "HU"), - **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), - **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", - "torino", "bologna", "florence"), "IT"), - **dict.fromkeys(("latvia", "riga"), "LV"), - **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), - **dict.fromkeys(("luxembourg",), "LU"), - **dict.fromkeys(("malta", "valletta"), "MT"), - **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", - "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", +#: Letters NFKD leaves whole because they are letters of their own rather +#: than a base letter with a mark, spelt the way boards write them in ASCII. +_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe", + "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"}) + + +def _place_fold(text: str) -> str: + """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København" + meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is, + since it also builds `company_key`, the primary key of `companies`.""" + return _fold((text or "").casefold().translate(_LETTERS)) + + +#: Every ISO 3166-1 country by its English name, plus the names boards also +#: write for it: its own language's for a European one, and the usual short +#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one. +_COUNTRY_NAMES = """ +AD Andorra +AE United Arab Emirates | UAE | U.A.E. | Emirates +AF Afghanistan +AG Antigua and Barbuda +AI Anguilla +AL Albania | Shqipëria | Shqipëri +AM Armenia | Hayastan +AO Angola +AQ Antarctica +AR Argentina +AS American Samoa +AT Austria | Österreich +AU Australia +AW Aruba +AX Åland Islands | Åland +AZ Azerbaijan | Azərbaycan +BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina +BB Barbados +BD Bangladesh +BE Belgium | België | Belgique | Belgien +BF Burkina Faso +BG Bulgaria | България | Balgariya +BH Bahrain +BI Burundi +BJ Benin +BL Saint Barthélemy +BM Bermuda +BN Brunei | Brunei Darussalam +BO Bolivia +BQ Caribbean Netherlands | Bonaire +BR Brazil | Brasil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus | Беларусь | Belarus' +BZ Belize +CA Canada +CC Cocos Islands | Cocos (Keeling) Islands +CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa +CF Central African Republic +CG Republic of the Congo | Congo-Brazzaville +CH Switzerland | Schweiz | Suisse | Svizzera | Svizra +CI Côte d'Ivoire | Ivory Coast +CK Cook Islands +CL Chile +CM Cameroon +CN China | People's Republic of China +CO Colombia +CR Costa Rica +CU Cuba +CV Cabo Verde | Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus | Κύπρος | Kypros | Kibris +CZ Czechia | Czech Republic | Česko | Česká republika +DE Germany | Deutschland +DJ Djibouti +DK Denmark | Danmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia | Eesti +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain | España | Espanya +ET Ethiopia +FI Finland | Suomi +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands | Føroyar | Færøerne +FR France +GA Gabon +GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland +GD Grenada +GE Georgia | Sakartvelo +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece | Ελλάδα | Hellas | Ellada +GS South Georgia and the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island and McDonald Islands +HN Honduras +HR Croatia | Hrvatska +HT Haiti +HU Hungary | Magyarország +ID Indonesia +IE Ireland | Éire +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland | Ísland +IT Italy | Italia +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN Saint Kitts and Nevis +KP North Korea +KR South Korea | Korea | Republic of Korea +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic +LB Lebanon +LC Saint Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania | Lietuva +LU Luxembourg | Lëtzebuerg | Luxemburg +LV Latvia | Latvija +LY Libya +MA Morocco +MC Monaco +MD Moldova | Republic of Moldova +ME Montenegro | Crna Gora +MF Saint Martin +MG Madagascar +MH Marshall Islands +MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија +ML Mali +MM Myanmar | Burma +MN Mongolia +MO Macao | Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico | México +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands | The Netherlands | Nederland | Holland +NO Norway | Norge | Noreg +NP Nepal +NR Nauru +NU Niue +NZ New Zealand | Aotearoa +OM Oman +PA Panama | Panamá +PE Peru | Perú +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland | Polska +PM Saint Pierre and Miquelon +PN Pitcairn Islands +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania | România +RS Serbia | Srbija | Србија +RU Russia | Russian Federation | Россия | Rossiya +RW Rwanda +SA Saudi Arabia | KSA +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden | Sverige +SG Singapore +SH Saint Helena +SI Slovenia | Slovenija +SJ Svalbard and Jan Mayen | Svalbard +SK Slovakia | Slovensko +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST São Tomé and Príncipe +SV El Salvador +SX Sint Maarten +SY Syria +SZ Eswatini | Swaziland +TC Turks and Caicos Islands +TD Chad +TF French Southern Territories +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL Timor-Leste | East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey | Türkiye +TT Trinidad and Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine | Україна | Ukraina +UG Uganda +UM United States Minor Outlying Islands +US United States | United States of America | USA | U.S.A. | US | U.S. +UY Uruguay +UZ Uzbekistan +VA Vatican City | Holy See +VC Saint Vincent and the Grenadines +VE Venezuela +VG British Virgin Islands +VI U.S. Virgin Islands | US Virgin Islands +VN Vietnam | Viet Nam +VU Vanuatu +WF Wallis and Futuna +WS Samoa +XK Kosovo | Kosova +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe +""" + +#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK. +_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line) + +#: The countries `EU` stands for when a line names several: the UN +#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and +#: the Council of Europe's members in the Caucasus and Turkey, which +#: European employers list beside their other offices. +_EUROPE = frozenset(( + "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE", + "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE", + "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL", + "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA", + "XK", +)) +_EUROPE_OR_EU = _EUROPE | {"EU"} + +#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them. +_REGIONS = ("europe", "emea", "eu", "european union", "eea") + +#: The cities boards name most, and a few regions below a country. A US +#: state or a Canadian province in the same part of the line beats one of +#: these ("London, Ontario" is Canada); a country name does not lose to it. +_CITIES: dict[str, str] = { + **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"), + **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"), + **dict.fromkeys(("sofia",), "BG"), + **dict.fromkeys(("zagreb",), "HR"), + **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), + **dict.fromkeys(("prague", "praha", "brno"), "CZ"), + **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", + "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", + "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", + "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), + **dict.fromkeys(("tallinn", "tartu"), "EE"), + **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", + "lille", "rennes", "amiens"), "FR"), + **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", + "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", + "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", + "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", + "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", + "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", + "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"), + **dict.fromkeys(("athens", "thessaloniki"), "GR"), + **dict.fromkeys(("budapest",), "HU"), + **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), + **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", + "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"), + **dict.fromkeys(("riga",), "LV"), + **dict.fromkeys(("vilnius", "kaunas"), "LT"), + **dict.fromkeys(("valletta",), "MT"), + **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), - **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", - "gdansk", "poznan", "lodz"), "PL"), - **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), - "RO"), - **dict.fromkeys(("slovakia", "bratislava"), "SK"), - **dict.fromkeys(("slovenia", "ljubljana"), "SI"), - **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", - "malaga", "bilbao"), "ES"), - **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", - "uppsala"), "SE"), - **dict.fromkeys(("iceland", "reykjavik"), "IS"), - **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), - **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", - "wales", "northern ireland", "london", "manchester", "edinburgh", - "glasgow", "bristol", "cambridge uk", "oxford", "leeds", - "birmingham uk", "belfast"), "GB"), - **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", - "basel", "lausanne", "bern", "zug"), "CH"), - **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", - "nyc", "san francisco", "seattle", "boston", "chicago", "austin", - "los angeles", "palo alto", "mountain view", "san mateo", - "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", - "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", - "houston", "phoenix", "tempe", "portland", "salt lake city", - "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", - "nashville", "minneapolis"), "US"), - **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", - "waterloo"), "CA"), - **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", - "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), + "PL"), + **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), + **dict.fromkeys(("belgrade", "beograd"), "RS"), + **dict.fromkeys(("bratislava",), "SK"), + **dict.fromkeys(("ljubljana",), "SI"), + **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", + "granada", "santiago de compostela"), "ES"), + **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), + **dict.fromkeys(("reykjavik",), "IS"), + **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), + **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", + "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", + "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", + "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", + "slough", "southampton", "swindon", "liverpool", "redhill", + "milton keynes"), "GB"), + **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", + "sankt gallen", "st gallen"), "CH"), + **dict.fromkeys(("kyiv", "kiev"), "UA"), + **dict.fromkeys(("istanbul", "ankara"), "TR"), + **dict.fromkeys(("tirana",), "AL"), + **dict.fromkeys(("yerevan",), "AM"), + **dict.fromkeys(("tbilisi",), "GE"), + **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", + "boston", "chicago", "austin", "los angeles", "palo alto", + "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", + "menlo park", "redwood city", "costa mesa", "denver", "atlanta", + "washington dc", "washington d c", "miami", "dallas", "houston", + "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", + "philadelphia", "detroit", "san diego", "raleigh", "nashville", + "minneapolis", "cambridge ma", "bay area", "arlington"), "US"), + **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), + "CA"), + **dict.fromkeys(("buenos aires",), "AR"), + **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"), + **dict.fromkeys(("santiago", "santiago de chile"), "CL"), + **dict.fromkeys(("bogota",), "CO"), + **dict.fromkeys(("lima", "miraflores"), "PE"), + **dict.fromkeys(("montevideo",), "UY"), + **dict.fromkeys(("managua",), "NI"), + **dict.fromkeys(("santo domingo",), "DO"), + **dict.fromkeys(("san salvador",), "SV"), + **dict.fromkeys(("panama city",), "PA"), + **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("dubai", "abu dhabi"), "AE"), + **dict.fromkeys(("riyadh",), "SA"), + **dict.fromkeys(("doha",), "QA"), + **dict.fromkeys(("manama",), "BH"), + **dict.fromkeys(("salmiya",), "KW"), + **dict.fromkeys(("muscat",), "OM"), + **dict.fromkeys(("cairo", "maadi"), "EG"), + **dict.fromkeys(("casablanca",), "MA"), + **dict.fromkeys(("dakar",), "SN"), + **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("almaty",), "KZ"), + **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), + **dict.fromkeys(("dhaka",), "BD"), + **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", + "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"), **dict.fromkeys(("singapore",), "SG"), - **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), - **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), - **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), - **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), - **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), - **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), - **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), + **dict.fromkeys(("manila", "taguig"), "PH"), + **dict.fromkeys(("bangkok",), "TH"), + **dict.fromkeys(("phnom penh",), "KH"), + **dict.fromkeys(("taipei",), "TW"), + **dict.fromkeys(("tokyo", "osaka"), "JP"), + **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"), + **dict.fromkeys(("seoul",), "KR"), + **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", + "queensland"), "AU"), } -#: Keyed the way a place is folded before it is looked up, so a name can be -#: written as it is spelt. -_PLACES = {_fold(name): code for name, code in _PLACES.items()} +#: US states and Canadian provinces, by name and by the code boards write +#: after a comma ("Boston, MA", "Toronto, ON"). +_SUBDIVISIONS = """ +US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO +US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA +US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY +US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN +US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH +US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND +US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI +US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT +US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY +CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB +CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL +CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT +CA Nunavut NU +""" -#: The longest name first, so "new york city" wins over "york" and "northern -#: ireland" over "ireland". -_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) -#: US state codes after a comma, the way US boards write a city: "Boston, MA". -#: Read only when no name matched, since "DE" and "IN" are countries too. -_US_STATE = re.compile( - r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" - r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" - r"\b") +def _subdivisions() -> tuple[dict[str, str], dict[str, str]]: + names: dict[str, str] = {} + codes: dict[str, str] = {} + for line in _SUBDIVISIONS.split("\n"): + if line: + country, rest = line.split(" ", 1) + for entry in rest.split("|"): + name, code = entry.strip().rsplit(" ", 1) + names[_place_fold(name)] = country + codes[code] = country + return names, codes + + +_STATE_NAMES, _STATE_CODES = _subdivisions() + + +def _names() -> dict[str, tuple[str, str]]: + """Every folded name to `(code, kind)`; kind is `country`, `city` or + `state`. A name that is a city and a state ("New York") is the city, so + it does not beat another city in its line. A name that is a country and + a state ("Georgia") is the country here and read as the state only when + the line names a place in the US too.""" + names: dict[str, tuple[str, str]] = {} + for name, code in _STATE_NAMES.items(): + names[name] = (code, "state") + for name, code in _CITIES.items(): + names[_place_fold(name)] = (code, "city") + for line in _COUNTRY_NAMES.split("\n"): + if line: + code, rest = line.split(" ", 1) + for name in rest.split("|"): + names[_place_fold(name)] = (code, "country") + for name in _REGIONS: + names[name] = ("EU", "country") + return names + + +_NAMES = _names() + +#: A state name that is also a country's, read as the state only when the +#: line names a place in that state's country too. +_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES + if _NAMES[name][1] == "country"} + +#: The longest name in words, so a lookup tries "new york city" before "new +#: york" and "northern ireland" before "ireland". +_LONGEST = max(len(name.split()) for name in _NAMES) + +#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin". +_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b") + +#: A country code in parentheses, "Leicester (GB)", read only when no name matched. +_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)") + +#: Where one location in a line ends and the next begins. A state beats a +#: city only within one: "London; Ontario" is two places. +_SEGMENT = re.compile(r"[;|/\n]") + + +def _hits(folded: str) -> list[tuple[str, str, str]]: + """The names in a folded line, longest first at each word, as + `(name, code, kind)`.""" + words = folded.split() + hits = [] + i = 0 + while i < len(words): + for n in range(min(_LONGEST, len(words) - i), 0, -1): + name = " ".join(words[i:i + n]) + if name in _NAMES: + hits.append((name, *_NAMES[name])) + i += n + break + else: + i += 1 + return hits def country_of(place: str) -> str: - """The one country a board's location line names, as an ISO code, or '' - when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is - US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" - is EU, and "London, EMEA" is GB: a country beats the region it is in.""" - folded = f" {_fold(place or '')} " + """The one country a board's location line names, as an ISO code; `EU` + when it names several and all of them are in Europe; '' when it names + none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston, + MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or + New York" and "Remote, EU or US" are ''. + + Within one location (the line split at `;`, `|` and `/`) the widest unit + named wins: a country beats a state or a province, which beats a city. + So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A + country also beats the region it is in: "London, EMEA" is GB. + + A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", + count as the state only when the line names a place in the US too: + "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather + than Delaware. A country code in parentheses, "Leicester (GB)", is read + only when no name matched.""" + segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] + named = {code for _, hits in segments for _, code, _ in hits} found: set[str] = set() - for name in _PLACE_NAMES: - needle = f" {name} " - if needle in folded: - found.add(_PLACES[name]) - folded = folded.replace(needle, " ") - if not found and _US_STATE.search(place or ""): - # Only when nothing else named a country: "Berlin, DE" is Germany. - found.add("US") - if len(found) > 1: - found.discard("EU") - return found.pop() if len(found) == 1 else "" + for text, hits in segments: + shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named} + countries = {code for name, code, kind in hits + if kind == "country" and code != "EU" and name not in shared} + states = {code for _, code, kind in hits if kind == "state"} + states |= {_SHARED_NAMES[name] for name in shared} + states |= {country for code in _STATE_CODE.findall(text) + if (country := _STATE_CODES.get(code)) + and (code not in _ISO or country in named)} + found |= (countries or states or {code for _, code, _ in hits if code != "EU"}) + found |= {code for _, code, _ in hits if code == "EU"} + if not found: + found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO} + if len(found) == 1: + return found.pop() + if found and found <= _EUROPE_OR_EU: + countries = found - {"EU"} + return countries.pop() if "EU" in found and len(countries) == 1 else "EU" + return "" # --------------------------------------------------------------- vocabulary diff --git a/app/storage/schema.py b/app/storage/schema.py index 3219e87..5f6b2e7 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -17,7 +17,7 @@ import logging import sqlite3 from collections.abc import Callable -from app.models import posted_at, posted_host +from app.models import country_of, posted_at, posted_host log = logging.getLogger(__name__) @@ -381,6 +381,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; # its current rules is the whole point (see _recompute_function below). (11, "roles.function recomputed for every not-yet-extracted role, word-boundary " "classifier fix", lambda conn: _recompute_function(conn)), + (12, "roles.country recomputed from place for every not-yet-extracted role whose " + "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -429,6 +431,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None: len(updates), len(rows)) +def _recompute_country(conn: sqlite3.Connection) -> None: + """Migration 12's step. `app.models.country_of` used to read a US state + code over a country's ("Bogota, CO" was US), return '' for a line naming + several European countries, and miss most countries outside Europe; + `Store.backfill_from_place` fills only an empty column, so a wrong code + would stay. Recomputed from `place` for every role `extract` has not + read (its answer only ever filled an empty column, but a role it read is + left as it is) and whose board sent no `country` of its own in `raw` + (the `search` plugin's Jobindex and Adzuna rows do), which is what + `Store.upsert_role` would write for that role today.""" + rows = conn.execute( + "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall() + updates = [] + for uid, place, raw, old in rows: + try: + given = json.loads(raw or "{}").get("country") or "" + except (TypeError, ValueError, AttributeError): + given = "" + if given: + continue + new = country_of(place or "") + if new != old: + updates.append((new, uid)) + if updates: + conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates) + log.info("migration 12: recomputed country for %d of %d not-yet-extracted roles", + len(updates), len(rows)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index d95264f..d018147 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -42,7 +42,7 @@ Advanced + Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: [{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}] is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.
diff --git a/docs/decisions.md b/docs/decisions.md index af07529..41736ae 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -32,7 +32,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. -**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. +**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the location line to an ISO code from every ISO country by its English name (and its own-language name for a European one) and the cities the boards name most, where `extract` would be a paid call per role for the same answer. On the live data (2026-09-29) 7.0% of open roles resolve to nothing, down from 16.8%, and most of those write only "Hybrid", "Remote" or a region such as "Americas". The line is folded by `_place_fold`, which also maps ł, đ, ø and æ since NFKD leaves them whole; `_fold` stays as it is because it builds `company_key`. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a US state or a Canadian province, which beats a city, so "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A state code that is also a country's (DE, CO, MA, IN) counts as the state only when the line also names a US place, so "Bogota, CO" is CO and "Paderborn, DE" is ''. A line naming several countries is `EU` when all of them are European (the UN's Europe, plus Cyprus, Turkey and the Council of Europe's Caucasus members) and '' otherwise: "Dublin, London" is EU, "Remote, EU or US" is ''. `EU`, ISO's reserved code, also stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it; a country beats the region it is in ("London, EMEA" is GB). `backfill_from_place` only fills an empty column, so a fix that changes answers comes with a migration: v12 recomputes `country` for every role `extract` has not read and whose board sent no `country` in `raw`, which changed 1,705 open roles, 48 of them from one code to another or to '' (25 Hong Kong roles moved from CN to HK). **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. diff --git a/tests/test_search.py b/tests/test_search.py index a18f404..813bff4 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store: ("London or New York", ""), ("Remote", ""), ("", ""), + ("Wrocław", "PL"), # ł and đ do not decompose under NFKD + ("Łódź, Poland", "PL"), + ("Bogota, CO", "CO"), # a state code that is a country's too + ("Paderborn, DE", ""), # ... is no US state without a US place + ("Sliema, MT", ""), + ("Casablanca, MA", "MA"), + ("Cambridge, MA", "US"), + ("Chicago, IL", "US"), + ("Madrid, MD, Spain", "ES"), + ("Dublin, London", "EU"), # several European countries + ("Anywhere in France, Belgium, Spain", "EU"), + ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), + ("Remote, EU or US", ""), # a region and a country outside it + ("London, Ontario", "CA"), # a province beats the city before it + ("Bellevue, Washington", "US"), + ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code + ("Atlanta, Georgia", "US"), + ("Tbilisi, Georgia", "GE"), + ("Malaysia", "MY"), + ("Belgrade, Serbia", "RS"), + ("Dubai", "AE"), + ("United Arab Emirates", "AE"), + ("Buenos Aires, , Argentina", "AR"), + ("Leicester (GB)", "GB"), # a code in parentheses when nothing else + ("Cagliari (CA)", "IT"), # ... and not over a name + ("Hybrid", ""), + ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code diff --git a/tests/test_storage.py b/tests/test_storage.py index 0fb2a8a..0de97c4 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 11 + assert LATEST == 12 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -257,6 +257,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen +def test_v12_recomputes_country_from_place_unless_the_board_or_extract_set_it( + tmp_path: Path, clock: Clock) -> None: + path = tmp_path / "v11.db" + store = Store(path) + store.clock = clock + wrong = posting("1", place="Bogota, CO") # was read as US + several = posting("2", place="Dublin, London") # was '' + given = posting("3", place="Bogota, CO", country="US") # the board said so + extracted = posting("4", place="Bogota, CO") + for item in (wrong, several, given, extracted): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") + conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) + conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", + (extracted.uid,)) + conn.execute("PRAGMA user_version = 11") + + migrated = Store(path) + + assert migrated.migrations_applied == list(range(12, LATEST + 1)) + countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid) + if (role := migrated.role(uid)) is not None} + assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US", + extracted.uid: "US"} + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23"