From 3e3060ba50976e889b8d5f2f87c5946edb56e5bc Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 13:34:27 +0200 Subject: [PATCH] Link the obvious towns of the Unmapped list: 460 unlinked place texts down to 50 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About 190 German towns (mostly 1KOMMA5°'s) and the German states, French, Italian, Dutch, Asian and other towns, the ISO3 codes and German or French country names boards write. Towns keep their marks through spelling groups. The tests that used Paderborn, Jena and Überlingen as unknown towns use Bopfingen, Lauchheim and Ellwangen instead. Left: names that are no place, ambiguous ones (Halifax, Richmond, Victoria, Durham, Warwick) and bare codes. Co-Authored-By: Claude Opus 5.5 --- app/models.py | 186 +++++++++++++++++++++++++++------------- app/places.py | 48 +++++++++-- tests/test_gazetteer.py | 20 ++--- tests/test_places.py | 3 +- tests/test_prep.py | 14 +-- tests/test_search.py | 2 +- 6 files changed, 188 insertions(+), 85 deletions(-) diff --git a/app/models.py b/app/models.py index 571c2d7..715a986 100644 --- a/app/models.py +++ b/app/models.py @@ -135,7 +135,7 @@ CC Cocos Islands | Cocos (Keeling) Islands CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa CF Central African Republic CG Republic of the Congo | Congo-Brazzaville -CH Switzerland | Schweiz | Suisse | Svizzera | Svizra +CH Switzerland | Schweiz | Suisse | Svizzera | Svizra | CHE CI Côte d'Ivoire | Ivory Coast CK Cook Islands CL Chile @@ -151,7 +151,7 @@ CY Cyprus | Κύπρος | Kypros | Kibris CZ Czechia | Czech Republic | Česko | Česká republika DE Germany | Deutschland | DEU | Deutschlandweit DJ Djibouti -DK Denmark | Danmark +DK Denmark | Danmark | Dänemark DM Dominica DO Dominican Republic DZ Algeria @@ -160,16 +160,16 @@ EE Estonia | Eesti EG Egypt EH Western Sahara ER Eritrea -ES Spain | España | Espanya +ES Spain | España | Espanya | ESP | Spanien | Espagne ET Ethiopia FI Finland | Suomi FJ Fiji FK Falkland Islands FM Micronesia FO Faroe Islands | Føroyar | Færøerne -FR France +FR France | FRA GA Gabon -GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland +GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland | GBR GD Grenada GE Georgia | Sakartvelo GF French Guiana @@ -187,7 +187,7 @@ GT Guatemala GU Guam GW Guinea-Bissau GY Guyana -HK Hong Kong +HK Hong Kong | 香港 HM Heard Island and McDonald Islands HN Honduras HR Croatia | Hrvatska @@ -206,7 +206,7 @@ IT Italy | Italia JE Jersey JM Jamaica JO Jordan -JP Japan +JP Japan | JPN KE Kenya KG Kyrgyzstan KH Cambodia @@ -249,8 +249,8 @@ MT Malta MU Mauritius MV Maldives MW Malawi -MX Mexico | México -MY Malaysia +MX Mexico | México | MEX +MY Malaysia | MYS MZ Mozambique NA Namibia NC New Caledonia @@ -258,7 +258,7 @@ NE Niger NF Norfolk Island NG Nigeria NI Nicaragua -NL Netherlands | The Netherlands | Nederland | Holland +NL Netherlands | The Netherlands | Nederland | Holland | NLD | Niederlande NO Norway | Norge | Noreg NP Nepal NR Nauru @@ -285,11 +285,11 @@ RO Romania | România RS Serbia | Srbija | Србија RU Russia | Russian Federation | Россия | Rossiya RW Rwanda -SA Saudi Arabia | KSA +SA Saudi Arabia | KSA | SAU SB Solomon Islands SC Seychelles SD Sudan -SE Sweden | Sverige +SE Sweden | Sverige | Suède SG Singapore SH Saint Helena SI Slovenia | Slovenija @@ -310,7 +310,7 @@ TC Turks and Caicos Islands TD Chad TF French Southern Territories TG Togo -TH Thailand +TH Thailand | THA TJ Tajikistan TK Tokelau TL Timor-Leste | East Timor @@ -368,8 +368,8 @@ _REGIONS = ("europe", "emea", "eu", "european union", "eea") #: state or a Canadian province in the same part of the line beats one of #: these ("London, Ontario" is Canada); a country name does not lose to it. _CITIES: dict[str, str] = { - **dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck", "villach"), - "AT"), + **dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck", "villach", + "klagenfurt"), "AT"), **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege", "luik"), "BE"), **dict.fromkeys(("sofia",), "BG"), @@ -378,73 +378,129 @@ _CITIES: dict[str, str] = { **dict.fromkeys(("prague", "praha", "brno"), "CZ"), **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", - "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", - "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), + "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", "brondby", + "glostrup", "soborg", "malov", "birkerod", "hornbaek"), "DK"), **dict.fromkeys(("tallinn", "tartu"), "EE"), **dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"), - **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", - "lille", "rennes", "amiens"), "FR"), + **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", "lille", + "rennes", "amiens", "rouen", "aix en provence", "grenoble", "annecy", "metz", + "strasbourg", "antibes", "avignon", "bourg en bresse", "brest", "caen", + "chambery", "colmar", "montpellier", "nancy", "nice", "nimes", + "saint etienne", "saint nazaire", "saint ouen l aumone", "valence", "angers", + "bayonne", "beauvais", "boulogne sur mer", "challans", "cholet", + "clermont ferrand", "dunkerque", "la roche sur yon", "la rochelle", + "lamballe", "lannion", "laval", "le havre", "le mans", "lens", + "les sables d olonne", "limoges", "lorient", "loudeac", "morlaix", "mulhouse", + "orleans", "orvault", "paimpol", "pornic", "quimper", "reims", "reze", + "saint brieuc", "saint herblain", "saint malo", "tours", "troyes", "vannes", + "vertou"), "FR"), **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", - "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", - "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck", + "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", "oldenburg", + "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck", "bayreuth", "bad homburg", "homburg", "ettlingen", "bremerhaven", "breidenbach", "bielefeld", "saarbrucken", "hildesheim", "moers", "kiel", - "ulm", "regensburg", "freiburg", "obersulm", "seevetal", - "neukirchen vluyn"), "DE"), + "ulm", "regensburg", "freiburg", "obersulm", "seevetal", "neukirchen vluyn", + "flensburg", "ingolstadt", "kempten", "konstanz", "magdeburg", "weimar", + "chemnitz", "erfurt", "freiberg", "gustrow", "kaiserslautern", "mohnesee", + "neubrandenburg", "neumarkt", "neumunster", "paderborn", "pforzheim", + "aachen", "amberg", "annaberg buchholz", "backnang", "bad kissingen", + "bad liebenzell", "baden baden", "bamberg", "besigheim", + "bietigheim bissingen", "bocholt", "brackenheim", "bretten", "bruchsal", + "cloppenburg", "cottbus", "crailsheim", "dachau", "darmstadt", + "donaueschingen", "donauworth", "ebersberg", "eppingen", "erding", "erlangen", + "ettenheim", "feuchtwangen", "filderstadt", "florsheim", "flohrsheim", + "freising", "friedrichshafen", "fulda", "furth", "geislingen", "gera", + "germering", "goppingen", "gundelsheim", "gunzburg", "hagenow", "halle saale", + "heidenheim", "heilbronn", "herrsching", "herzogenaurach", "holzkirchen", + "holzminden", "hoxter", "idar oberstein", "jena", "kaisheim", "kierspe", + "kirchheim", "kitzingen", "koblenz", "kolbermoor", "konigsbrunn", "kunzelsau", + "landau", "landsberg", "landshut", "lauf an der pegnitz", "lindern", "lingen", + "lorrach", "ludwigsburg", "ludwigshafen", "malchow", "marburg", "meissen", + "memmingen", "meppen", "metzingen", "monchengladbach", "moosburg", "mosbach", + "muhldorf", "mulheim", "murnau", "neuburg an der donau", "neustadt", + "nordlingen", "northeim", "oberhausen", "offenburg", "ohringen", "passau", + "peine", "petershausen", "pfaffenhofen", "pfullendorf", "plau am see", + "prien am chiemsee", "rastatt", "reutlingen", "riesa", "robel", "roth", + "rothenburg ob der tauber", "rottweil", "salzgitter", "sankt georgen", + "schorndorf", "schwabisch gmund", "schwabisch hall", "schweinfurt", + "starnberg", "stockach", "tegernsee", "templin", "trier", "tuttlingen", + "uberlingen", "uedem", "vaihingen", "villingen schwenningen", + "waren an der muritz", "wasserburg", "weilheim", "wernigerode", "wismar", + "wittenberg", "wittstock", "wolfenbuttel", "wolfratshausen", "wolfsburg", + "zwickau", "euskirchen", "tangstedt", "erkrath", "sigmaringen", "grimmen", + "ribnitz damgarten", "stralsund", "warstein", "ansbach", "balingen", "coburg", + "duisburg", "grossostheim", "merzig", "saarlouis", "schwerin", "singen", + "troisdorf", "bad bevensen", "dannenberg", "luchow", "nagold", "salzwedel", + "uelzen", "boblingen", "giessen", "hagen", "hechingen", "leverkusen", + "solingen", "wetzlar"), "DE"), **dict.fromkeys(("athens", "thessaloniki"), "GR"), - **dict.fromkeys(("budapest",), "HU"), + **dict.fromkeys(("budapest", "cegled"), "HU"), **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), - **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", - "florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples", - "napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"), + **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", "florence", + "verona", "cagliari", "ascoli piceno", "ancarano", "naples", "napoli", + "venice", "venezia", "genoa", "genova", "bolzano", "padua", "padova", + "brindisi", "matera", "pistoia", "taranto", "asti", "campobasso", "cassino", + "chieti", "civitavecchia", "cosenza", "crema", "fiumicino", "frosinone", + "grosseto", "guidonia montecelio", "l aquila", "ladispoli", "lamezia terme", + "latina", "livorno", "lucca", "massa carrara", "pescara", "pisa", "ravenna", + "rimini", "sanremo", "teramo", "terni", "trieste", "vibo valentia", "forli", + "cesena", "emilia romagna", "tuscany", "lazio"), "IT"), **dict.fromkeys(("riga",), "LV"), **dict.fromkeys(("vilnius", "kaunas"), "LT"), **dict.fromkeys(("luxembourg", "luxemburg"), "LU"), **dict.fromkeys(("valletta",), "MT"), **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", - "delft"), "NL"), - **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), - "PL"), + "delft", "amersfoort", "deventer", "maastricht", "bleiswijk", "almere", + "enschede"), "NL"), + **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz", + "biskupiec", "sochaczew", "teresin"), "PL"), **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara", + "brasov"), "RO"), **dict.fromkeys(("belgrade", "beograd"), "RS"), **dict.fromkeys(("bratislava",), "SK"), **dict.fromkeys(("ljubljana",), "SI"), - **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", - "granada", "santiago de compostela"), "ES"), + **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", "granada", + "santiago de compostela", "pontevedra", "tarragona"), "ES"), **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras", - "linkoping", "orebro", "norrkoping", "jonkoping"), "SE"), + "linkoping", "orebro", "norrkoping", "jonkoping", "almhult", "halmstad", + "solna"), "SE"), **dict.fromkeys(("reykjavik",), "IS"), **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", - "slough", "southampton", "swindon", "liverpool", "redhill", - "milton keynes"), "GB"), + "slough", "southampton", "swindon", "liverpool", "redhill", "milton keynes", + "birmingham", "cambridge", "fermanagh", "tyrone"), "GB"), **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", - "sankt gallen", "st gallen", "lucerne", "luzern"), "CH"), + "sankt gallen", "st gallen", "lucerne", "luzern", "landquart"), "CH"), **dict.fromkeys(("kyiv", "kiev"), "UA"), **dict.fromkeys(("istanbul", "ankara"), "TR"), **dict.fromkeys(("tirana",), "AL"), **dict.fromkeys(("yerevan",), "AM"), **dict.fromkeys(("tbilisi",), "GE"), **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", - "boston", "chicago", "austin", "los angeles", "palo alto", - "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", - "menlo park", "redwood city", "costa mesa", "denver", "atlanta", - "washington dc", "washington d c", "miami", "dallas", "houston", - "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", - "philadelphia", "detroit", "san diego", "raleigh", "nashville", - "minneapolis", "cambridge ma", "bay area", "arlington"), "US"), - **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), - "CA"), + "boston", "chicago", "austin", "los angeles", "palo alto", "mountain view", + "san mateo", "foster city", "sunnyvale", "san jose", "menlo park", + "redwood city", "costa mesa", "denver", "atlanta", "washington dc", + "washington d c", "miami", "dallas", "houston", "phoenix", "tempe", + "portland", "salt lake city", "pittsburgh", "philadelphia", "detroit", + "san diego", "raleigh", "nashville", "minneapolis", "cambridge ma", + "bay area", "arlington", "leominster", "andover", "el segundo", "columbus", + "tysons corner", "tysons", "lowell", "mclean", "morrisville", "southlake", + "st louis", "aventura", "brooklyn", "charlotte", "corte madera", + "morgan hill", "newport beach", "las vegas", "san antonio", "tampa", + "indianapolis", "honolulu", "memphis", "glenview", "shakopee", "lexington", + "wilmington", "murrieta"), "US"), + **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo", + "edmonton", "winnipeg"), "CA"), **dict.fromkeys(("buenos aires",), "AR"), - **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"), + **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos", "curitiba"), "BR"), **dict.fromkeys(("santiago", "santiago de chile"), "CL"), **dict.fromkeys(("bogota",), "CO"), **dict.fromkeys(("lima", "miraflores"), "PE"), @@ -453,7 +509,7 @@ _CITIES: dict[str, str] = { **dict.fromkeys(("santo domingo",), "DO"), **dict.fromkeys(("san salvador",), "SV"), **dict.fromkeys(("panama city",), "PA"), - **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara", "cdmx"), "MX"), **dict.fromkeys(("dubai", "abu dhabi"), "AE"), **dict.fromkeys(("riyadh",), "SA"), **dict.fromkeys(("doha",), "QA"), @@ -463,23 +519,31 @@ _CITIES: dict[str, str] = { **dict.fromkeys(("cairo", "maadi"), "EG"), **dict.fromkeys(("casablanca",), "MA"), **dict.fromkeys(("dakar",), "SN"), - **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa", "netanya"), "IL"), **dict.fromkeys(("almaty",), "KZ"), **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), **dict.fromkeys(("dhaka",), "BD"), - **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", - "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), - **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"), + **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", "gurgaon", + "gurugram", "noida", "new delhi", "delhi", "ahmedabad"), "IN"), + **dict.fromkeys(("kuala lumpur", "bangsar south", "melaka", "malacca", "kulim", "penang", + "ipoh"), "MY"), **dict.fromkeys(("singapore",), "SG"), - **dict.fromkeys(("manila", "taguig"), "PH"), - **dict.fromkeys(("bangkok",), "TH"), + **dict.fromkeys(("manila", "taguig", "cavite", "general trias", "muntinlupa"), "PH"), + **dict.fromkeys(("bangkok", "samut prakan", "samut prakarn"), "TH"), **dict.fromkeys(("phnom penh",), "KH"), - **dict.fromkeys(("taipei",), "TW"), - **dict.fromkeys(("tokyo", "osaka"), "JP"), - **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"), - **dict.fromkeys(("seoul",), "KR"), - **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", - "queensland"), "AU"), + **dict.fromkeys(("taipei", "hsinchu", "kaohsiung", "tainan"), "TW"), + **dict.fromkeys(("tokyo", "osaka", "nagoya"), "JP"), + **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou", "wuxi", "xi an", + "qingdao", "chongqing", "nanjing", "tianjin", "wuhan", "xiamen"), "CN"), + **dict.fromkeys(("seoul", "seongnam", "busan", "daegu", "daejeon", "gwangju"), "KR"), + **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", "queensland", + "canberra"), "AU"), + **dict.fromkeys(("jakarta", "batam"), "ID"), + **dict.fromkeys(("hanoi", "ho chi minh"), "VN"), + **dict.fromkeys(("tunis",), "TN"), + **dict.fromkeys(("ebene",), "MU"), + **dict.fromkeys(("cape town", "johannesburg"), "ZA"), + **dict.fromkeys(("auckland",), "NZ"), } #: US states and Canadian provinces, by name and by the code boards write @@ -592,7 +656,7 @@ def country_of(place: str) -> str: A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", count as the state only when the line names a place in the US too: - "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather + "Berlin, DE" is DE, "Bogota, CO" is CO, "Ellwangen, DE" is '' rather than Delaware. A country code in parentheses, "Leicester (GB)", is read only when no name matched.""" segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] diff --git a/app/places.py b/app/places.py index f443a06..a72564a 100644 --- a/app/places.py +++ b/app/places.py @@ -179,6 +179,27 @@ _ALIAS_GROUPS: tuple[tuple[str, ...], ...] = ( ("Timișoara", "Timisoara"), ("Marseille", "Marseilles"), ("Bolzano", "Bozen"), + ("Padua", "Padova"), + ("Forlì",), + ("L'Aquila",), + ("Massa-Carrara", "Massa Carrara"), + # France, with the marks and hyphens a fold drops + ("Aix-en-Provence",), ("Bourg-en-Bresse",), ("Boulogne-sur-Mer",), ("Chambéry",), + ("Clermont-Ferrand",), ("La Roche-sur-Yon",), ("Les Sables-d'Olonne",), ("Loudéac",), + ("Nîmes",), ("Orléans",), ("Rezé",), ("Saint-Brieuc",), ("Saint-Étienne",), + ("Saint-Herblain",), ("Saint-Malo",), ("Saint-Nazaire",), ("Saint-Ouen-l'Aumône",), + # Germany, likewise + ("Annaberg-Buchholz",), ("Baden-Baden",), ("Bietigheim-Bissingen",), ("Böblingen",), + ("Donauwörth",), ("Flörsheim", "Flohrsheim"), ("Fürth",), ("Gießen", "Giessen"), + ("Göppingen",), ("Großostheim",), ("Günzburg",), ("Güstrow",), ("Halle (Saale)",), + ("Höxter",), ("Idar-Oberstein",), ("Königsbrunn",), ("Künzelsau",), ("Lörrach",), + ("Lüchow",), ("Meißen", "Meissen"), ("Möhnesee",), ("Mönchengladbach",), ("Mühldorf",), + ("Mülheim",), ("Neumünster",), ("Nördlingen",), ("Öhringen",), ("Ribnitz-Damgarten",), + ("Röbel",), ("Schwäbisch Gmünd",), ("Schwäbisch Hall",), ("Überlingen",), + ("Villingen-Schwenningen",), ("Waren an der Müritz",), ("Wolfenbüttel",), + # Elsewhere + ("Cegléd",), ("Brașov", "Brasov"), ("Älmhult",), ("Hornbæk",), ("Melaka", "Malacca"), + ("Samut Prakan", "Samut Prakarn"), ("Xi'an",), ("Ho Chi Minh City", "Ho Chi Minh"), ) @@ -305,13 +326,30 @@ _ORESUND = Region( _split("Øresund, Öresund, Oresund, Oresund Region, Øresundsregionen, Öresundsregionen"), (), includes=("hovedstaden", "skane")) -# Germany's by the names boards write for a remote role's area, towns not -# listed: a town in Germany is found through the country. +# Germany's states and the areas boards write for a remote role's area, by +# name, towns not listed: a town in Germany is found through the country. _GERMANY = tuple(Region(key, label, _split(names), (), "DE") for key, label, names in ( ("niedersachsen", "Niedersachsen", "Niedersachsen, Lower Saxony"), ("norddeutschland", "Norddeutschland", "Norddeutschland, Northern Germany"), ("ostdeutschland", "Ostdeutschland", "Ostdeutschland, Eastern Germany"), - ("suddeutschland", "Süddeutschland", "Süddeutschland, Southern Germany"))) + ("suddeutschland", "Süddeutschland", "Süddeutschland, Southern Germany"), + ("baden-wurttemberg", "Baden-Württemberg", "Baden-Württemberg"), + ("bayern", "Bayern", "Bayern, Bavaria"), + ("brandenburg", "Brandenburg", "Brandenburg"), + ("hessen", "Hessen", "Hessen, Hesse"), + ("mecklenburg-vorpommern", "Mecklenburg-Vorpommern", + "Mecklenburg-Vorpommern, Mecklenburg-Western Pomerania"), + ("nordrhein-westfalen", "Nordrhein-Westfalen", "Nordrhein-Westfalen, North Rhine-Westphalia, NRW"), + ("rheinland-pfalz", "Rheinland-Pfalz", "Rheinland-Pfalz, Rhineland-Palatinate"), + ("saarland", "Saarland", "Saarland"), + ("sachsen", "Sachsen", "Sachsen, Saxony"), + ("sachsen-anhalt", "Sachsen-Anhalt", "Sachsen-Anhalt, Saxony-Anhalt"), + ("schleswig-holstein", "Schleswig-Holstein", "Schleswig-Holstein"), + ("thuringen", "Thüringen", "Thüringen, Thuringia"), + ("bodensee", "Bodensee", "Bodensee, Lake Constance"), + ("harz", "Harz", "Harz"), + ("rugen", "Rügen", "Rügen"), + ("uckermark", "Uckermark", "Uckermark"))) REGIONS: dict[str, Region] = { r.key: r for r in (_SKANE, _HOVEDSTADEN, _ZEALAND, _ORESUND, _JUTLAND, _FUNEN, *_GERMANY)} @@ -694,7 +732,7 @@ KINDS: tuple[str, ...] = ("continent", "country", "region", "city") #: Entries of `_CITIES` that name a region rather than a town. _CITY_REGIONS = frozenset(("new south wales", "queensland", "surrey", "sussex", "east anglia", - "bay area")) + "bay area", "emilia romagna", "tuscany", "lazio")) #: Words a place text holds when it names how or where-ever a role is worked #: rather than a place: "Remote", "Multiple Locations", "Home based". A text @@ -922,7 +960,7 @@ class Gazetteer: def resolve(self, text: str) -> tuple[int, ...]: """The places one place text names, most specific first from the left: `Aarhus N, Region Central Jutland, Denmark` is Aarhus, `Malmö/Lund` - is Malmö and Lund, `Paderborn, Germany` is Germany, `Remote` is none. + is Malmö and Lund, `Ellwangen, Germany` is Germany, `Remote` is none. Every alias the text holds as whole words is a candidate, longest first. A fold that names several places is the one in the country diff --git a/tests/test_gazetteer.py b/tests/test_gazetteer.py index 9b7f6da..24995a9 100644 --- a/tests/test_gazetteer.py +++ b/tests/test_gazetteer.py @@ -83,7 +83,7 @@ def test_copenhagen_is_in_the_capital_region_in_denmark_in_europe() -> None: ("Copenhagen Office", ["Copenhagen"]), ("Malmö/Lund", ["Malmö", "Lund"]), ("Aalborg, Copenhagen, Kolding", ["Aalborg", "Copenhagen", "Kolding"]), - ("Paderborn, Germany", ["Germany"]), + ("Bopfingen, Germany", ["Germany"]), ("Skåne län, Sweden", ["Skåne"]), ("London, Ontario", ["Ontario"]), ("Bergen op Zoom, Netherlands", ["Netherlands"]), @@ -117,7 +117,7 @@ def test_a_text_of_how_a_role_is_worked_is_no_place(text: str) -> None: assert places.placeless(text) -@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Jena"]) +@pytest.mark.parametrize("text", ["Oberkochen", "Remote, APAC", "Lauchheim"]) def test_a_text_with_a_name_is_a_place(text: str) -> None: assert not places.placeless(text) @@ -235,7 +235,7 @@ def uids(store: Store, *tokens: str) -> set[str]: def several(store: Store) -> dict[str, str]: lines = {"cph": "Copenhagen; London", "eu": "Berlin, Germany; Paris, France", "malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund", - "zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"} + "zeiss": "Oberkochen; Lauchheim", "remote": "Remote (SE)", "ny": "New York, NY"} for key, line in lines.items(): upsert(store, posting(key, place=line)) return {key: posting(key).uid for key in lines} @@ -312,10 +312,10 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store, # ------------------------------------------------------------ unresolved texts def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None: - for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")): + for n, line in enumerate(("Oberkochen; Lauchheim", "Oberkochen", "Remote", "Aalen; Berlin")): upsert(store, posting(str(n), place=line)) assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1), - ("jena", "Jena", 1)] + ("lauchheim", "Lauchheim", 1)] assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)] @@ -344,12 +344,12 @@ def test_an_alias_links_what_it_names(store: Store) -> None: def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None: - upsert(store, posting("1", place="Oberkochen; Jena")) - upsert(store, posting("2", place="Paderborn, Germany")) + upsert(store, posting("1", place="Oberkochen; Lauchheim")) + upsert(store, posting("2", place="Bopfingen, Germany")) oberkochen = store.add_place("Oberkochen", "city", "country:DE") - store.add_place("Paderborn", "city", "country:DE") - assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")] - assert texts(store, posting("2").uid) == [(0, "Paderborn, Germany", "Paderborn")] + store.add_place("Bopfingen", "city", "country:DE") + assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Lauchheim", "")] + assert texts(store, posting("2").uid) == [(0, "Bopfingen, Germany", "Bopfingen")] assert uids(store, "place:oberkochen") == {posting("1").uid} assert store.gazetteer().country(oberkochen) == "DE" with pytest.raises(ValueError): diff --git a/tests/test_places.py b/tests/test_places.py index f19dabd..b1d4944 100644 --- a/tests/test_places.py +++ b/tests/test_places.py @@ -351,7 +351,8 @@ def test_null_columns_match_nothing_and_do_not_raise() -> None: def test_every_region_has_names_and_towns() -> None: - germany = {"niedersachsen", "norddeutschland", "ostdeutschland", "suddeutschland"} + germany = {r.key for r in places._GERMANY} + assert {"niedersachsen", "bayern", "thuringen", "harz"} <= germany assert set(REGIONS) == {"skane", "hovedstaden", "zealand", "oresund", "jutland", "funen"} | germany for key, region in REGIONS.items(): diff --git a/tests/test_prep.py b/tests/test_prep.py index 462fc4e..92e2085 100644 --- a/tests/test_prep.py +++ b/tests/test_prep.py @@ -275,12 +275,12 @@ def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) def test_a_town_added_to_the_code_is_linked_at_the_next_prep( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: - store.upsert_role(item("1", places=[{"text": "Überlingen", "remote": None}])) + store.upsert_role(item("1", places=[{"text": "Ellwangen", "remote": None}])) prep.run(store, tables=TABLES) - assert [p["text"] for p in store.unlinked_places()] == ["Überlingen"] + assert [p["text"] for p in store.unlinked_places()] == ["Ellwangen"] assert row(store, "1")["country"] == "" - monkeypatch.setitem(models._CITIES, "uberlingen", "DE") + monkeypatch.setitem(models._CITIES, "ellwangen", "DE") done = prep.run(store, tables=TABLES) assert done.remapped assert store.unlinked_places() == [] @@ -340,9 +340,9 @@ def test_app_prep_runs_and_lists_the_unmapped_words( assert json.loads(capsys.readouterr().out)["read"] == 1 assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0 assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"] - Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Überlingen")) + Store(tmp_path / "roledesk.db").upsert_role(item("2", place="Ellwangen")) assert cli.main(["-c", str(path), "prep", "unlinked"]) == 0 - assert capsys.readouterr().out.split() == ["1", "Überlingen", "-:1", "Acme"] + assert capsys.readouterr().out.split() == ["1", "Ellwangen", "-:1", "Acme"] assert cli.main(["-c", str(path), "prep", "--remap"]) == 0 assert json.loads(capsys.readouterr().out)["remapped"] is True @@ -371,11 +371,11 @@ def test_the_api_lists_the_unlinked_places(tmp_path: Path) -> None: app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), database=tmp_path / "roledesk.db")) - app.state.store.upsert_role(item("1", place="Überlingen", country="DE")) + app.state.store.upsert_role(item("1", place="Ellwangen", country="DE")) prep.run(app.state.store, tables=TABLES) with TestClient(app) as client: found = client.get("/api/prep/unlinked").json() - assert found == [{"fold": "uberlingen", "text": "Überlingen", "roles": 1, + assert found == [{"fold": "ellwangen", "text": "Ellwangen", "roles": 1, "countries": [["DE", 1]], "companies": ["Acme"]}] diff --git a/tests/test_search.py b/tests/test_search.py index fdefec1..dc34a6a 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -60,7 +60,7 @@ def store(tmp_path: Path) -> Store: ("Wrocław", "PL"), # ł and đ do not decompose under NFKD ("Łódź, Poland", "PL"), ("Bogota, CO", "CO"), # a state code that is a country's too - ("Paderborn, DE", ""), # ... is no US state without a US place + ("Ellwangen, DE", ""), # ... is no US state without a US place ("Sliema, MT", ""), ("Casablanca, MA", "MA"), ("Cambridge, MA", "US"),