From 2c4b992c50758980f5b2b930c17848a6aa7095b7 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:06:15 +0200 Subject: [PATCH 1/4] Country lookup: every ISO country, the widest unit wins, several European countries are EU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit country_of folds with _place_fold (ł, đ, ø, æ map to ASCII; _fold keeps building company_key unchanged), knows every ISO country by English name and European ones by their own, and the cities among the 150 most frequent unresolved places on the live data. Within one location a country beats a US state or Canadian province, which beats a city. A state code that is also an ISO code counts as the state only when the line names a US place. Several countries, all European, are EU; otherwise ''. A code in parentheses is read when no name matched. Migration 12 recomputes country from place for roles extract has not read and whose board sent no country in raw. On the live data: open roles with no country 2,827 -> 1,183 of 16,841; 1,705 open roles change, 48 from a code. Co-Authored-By: Claude Opus --- app/models.py | 627 ++++++++++++++++++++++++----- app/storage/schema.py | 33 +- app/templates/settings/agents.html | 2 +- docs/decisions.md | 2 +- tests/test_search.py | 27 ++ tests/test_storage.py | 29 +- 6 files changed, 611 insertions(+), 109 deletions(-) diff --git a/app/models.py b/app/models.py index 46606d0..9a6b5db 100644 --- a/app/models.py +++ b/app/models.py @@ -66,121 +66,538 @@ def company_key(company: str) -> str: # ---------------------------------------------------------------- geography -#: Folded names a board writes in its location line, to an ISO code: every -#: European country in English and its own language, the cities the boards -#: name most, and the non-European places that fill the seed boards, so those -#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. -_PLACES: dict[str, str] = { - **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), - **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), - **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", - "ghent"), "BE"), - **dict.fromkeys(("bulgaria", "sofia"), "BG"), - **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), - **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), - **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), - # Folding keeps ø and æ, which are letters of their own rather than an o - # or an a with a mark, so København is listed beside Kobenhavn. - **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", - "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", - "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", - "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", - "birkerød"), "DK"), - **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), - **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), - **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", - "bordeaux", "lille"), "FR"), - **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", - "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", - "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", - "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), - **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), - **dict.fromkeys(("hungary", "budapest"), "HU"), - **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), - **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", - "torino", "bologna", "florence"), "IT"), - **dict.fromkeys(("latvia", "riga"), "LV"), - **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), - **dict.fromkeys(("luxembourg",), "LU"), - **dict.fromkeys(("malta", "valletta"), "MT"), - **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", - "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", +#: Letters NFKD leaves whole because they are letters of their own rather +#: than a base letter with a mark, spelt the way boards write them in ASCII. +_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe", + "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"}) + + +def _place_fold(text: str) -> str: + """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København" + meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is, + since it also builds `company_key`, the primary key of `companies`.""" + return _fold((text or "").casefold().translate(_LETTERS)) + + +#: Every ISO 3166-1 country by its English name, plus the names boards also +#: write for it: its own language's for a European one, and the usual short +#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one. +_COUNTRY_NAMES = """ +AD Andorra +AE United Arab Emirates | UAE | U.A.E. | Emirates +AF Afghanistan +AG Antigua and Barbuda +AI Anguilla +AL Albania | Shqipëria | Shqipëri +AM Armenia | Hayastan +AO Angola +AQ Antarctica +AR Argentina +AS American Samoa +AT Austria | Österreich +AU Australia +AW Aruba +AX Åland Islands | Åland +AZ Azerbaijan | Azərbaycan +BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina +BB Barbados +BD Bangladesh +BE Belgium | België | Belgique | Belgien +BF Burkina Faso +BG Bulgaria | България | Balgariya +BH Bahrain +BI Burundi +BJ Benin +BL Saint Barthélemy +BM Bermuda +BN Brunei | Brunei Darussalam +BO Bolivia +BQ Caribbean Netherlands | Bonaire +BR Brazil | Brasil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus | Беларусь | Belarus' +BZ Belize +CA Canada +CC Cocos Islands | Cocos (Keeling) Islands +CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa +CF Central African Republic +CG Republic of the Congo | Congo-Brazzaville +CH Switzerland | Schweiz | Suisse | Svizzera | Svizra +CI Côte d'Ivoire | Ivory Coast +CK Cook Islands +CL Chile +CM Cameroon +CN China | People's Republic of China +CO Colombia +CR Costa Rica +CU Cuba +CV Cabo Verde | Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus | Κύπρος | Kypros | Kibris +CZ Czechia | Czech Republic | Česko | Česká republika +DE Germany | Deutschland +DJ Djibouti +DK Denmark | Danmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia | Eesti +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain | España | Espanya +ET Ethiopia +FI Finland | Suomi +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands | Føroyar | Færøerne +FR France +GA Gabon +GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland +GD Grenada +GE Georgia | Sakartvelo +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece | Ελλάδα | Hellas | Ellada +GS South Georgia and the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island and McDonald Islands +HN Honduras +HR Croatia | Hrvatska +HT Haiti +HU Hungary | Magyarország +ID Indonesia +IE Ireland | Éire +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland | Ísland +IT Italy | Italia +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN Saint Kitts and Nevis +KP North Korea +KR South Korea | Korea | Republic of Korea +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic +LB Lebanon +LC Saint Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania | Lietuva +LU Luxembourg | Lëtzebuerg | Luxemburg +LV Latvia | Latvija +LY Libya +MA Morocco +MC Monaco +MD Moldova | Republic of Moldova +ME Montenegro | Crna Gora +MF Saint Martin +MG Madagascar +MH Marshall Islands +MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија +ML Mali +MM Myanmar | Burma +MN Mongolia +MO Macao | Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico | México +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands | The Netherlands | Nederland | Holland +NO Norway | Norge | Noreg +NP Nepal +NR Nauru +NU Niue +NZ New Zealand | Aotearoa +OM Oman +PA Panama | Panamá +PE Peru | Perú +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland | Polska +PM Saint Pierre and Miquelon +PN Pitcairn Islands +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania | România +RS Serbia | Srbija | Србија +RU Russia | Russian Federation | Россия | Rossiya +RW Rwanda +SA Saudi Arabia | KSA +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden | Sverige +SG Singapore +SH Saint Helena +SI Slovenia | Slovenija +SJ Svalbard and Jan Mayen | Svalbard +SK Slovakia | Slovensko +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST São Tomé and Príncipe +SV El Salvador +SX Sint Maarten +SY Syria +SZ Eswatini | Swaziland +TC Turks and Caicos Islands +TD Chad +TF French Southern Territories +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL Timor-Leste | East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey | Türkiye +TT Trinidad and Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine | Україна | Ukraina +UG Uganda +UM United States Minor Outlying Islands +US United States | United States of America | USA | U.S.A. | US | U.S. +UY Uruguay +UZ Uzbekistan +VA Vatican City | Holy See +VC Saint Vincent and the Grenadines +VE Venezuela +VG British Virgin Islands +VI U.S. Virgin Islands | US Virgin Islands +VN Vietnam | Viet Nam +VU Vanuatu +WF Wallis and Futuna +WS Samoa +XK Kosovo | Kosova +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe +""" + +#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK. +_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line) + +#: The countries `EU` stands for when a line names several: the UN +#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and +#: the Council of Europe's members in the Caucasus and Turkey, which +#: European employers list beside their other offices. +_EUROPE = frozenset(( + "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE", + "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE", + "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL", + "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA", + "XK", +)) +_EUROPE_OR_EU = _EUROPE | {"EU"} + +#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them. +_REGIONS = ("europe", "emea", "eu", "european union", "eea") + +#: The cities boards name most, and a few regions below a country. A US +#: state or a Canadian province in the same part of the line beats one of +#: these ("London, Ontario" is Canada); a country name does not lose to it. +_CITIES: dict[str, str] = { + **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"), + **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"), + **dict.fromkeys(("sofia",), "BG"), + **dict.fromkeys(("zagreb",), "HR"), + **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), + **dict.fromkeys(("prague", "praha", "brno"), "CZ"), + **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", + "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", + "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", + "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), + **dict.fromkeys(("tallinn", "tartu"), "EE"), + **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", + "lille", "rennes", "amiens"), "FR"), + **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", + "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", + "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", + "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", + "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", + "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", + "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"), + **dict.fromkeys(("athens", "thessaloniki"), "GR"), + **dict.fromkeys(("budapest",), "HU"), + **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), + **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", + "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"), + **dict.fromkeys(("riga",), "LV"), + **dict.fromkeys(("vilnius", "kaunas"), "LT"), + **dict.fromkeys(("valletta",), "MT"), + **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), - **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", - "gdansk", "poznan", "lodz"), "PL"), - **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), - "RO"), - **dict.fromkeys(("slovakia", "bratislava"), "SK"), - **dict.fromkeys(("slovenia", "ljubljana"), "SI"), - **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", - "malaga", "bilbao"), "ES"), - **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", - "uppsala"), "SE"), - **dict.fromkeys(("iceland", "reykjavik"), "IS"), - **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), - **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", - "wales", "northern ireland", "london", "manchester", "edinburgh", - "glasgow", "bristol", "cambridge uk", "oxford", "leeds", - "birmingham uk", "belfast"), "GB"), - **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", - "basel", "lausanne", "bern", "zug"), "CH"), - **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", - "nyc", "san francisco", "seattle", "boston", "chicago", "austin", - "los angeles", "palo alto", "mountain view", "san mateo", - "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", - "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", - "houston", "phoenix", "tempe", "portland", "salt lake city", - "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", - "nashville", "minneapolis"), "US"), - **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", - "waterloo"), "CA"), - **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", - "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), + "PL"), + **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), + **dict.fromkeys(("belgrade", "beograd"), "RS"), + **dict.fromkeys(("bratislava",), "SK"), + **dict.fromkeys(("ljubljana",), "SI"), + **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", + "granada", "santiago de compostela"), "ES"), + **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), + **dict.fromkeys(("reykjavik",), "IS"), + **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), + **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", + "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", + "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", + "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", + "slough", "southampton", "swindon", "liverpool", "redhill", + "milton keynes"), "GB"), + **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", + "sankt gallen", "st gallen"), "CH"), + **dict.fromkeys(("kyiv", "kiev"), "UA"), + **dict.fromkeys(("istanbul", "ankara"), "TR"), + **dict.fromkeys(("tirana",), "AL"), + **dict.fromkeys(("yerevan",), "AM"), + **dict.fromkeys(("tbilisi",), "GE"), + **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", + "boston", "chicago", "austin", "los angeles", "palo alto", + "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", + "menlo park", "redwood city", "costa mesa", "denver", "atlanta", + "washington dc", "washington d c", "miami", "dallas", "houston", + "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", + "philadelphia", "detroit", "san diego", "raleigh", "nashville", + "minneapolis", "cambridge ma", "bay area", "arlington"), "US"), + **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), + "CA"), + **dict.fromkeys(("buenos aires",), "AR"), + **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"), + **dict.fromkeys(("santiago", "santiago de chile"), "CL"), + **dict.fromkeys(("bogota",), "CO"), + **dict.fromkeys(("lima", "miraflores"), "PE"), + **dict.fromkeys(("montevideo",), "UY"), + **dict.fromkeys(("managua",), "NI"), + **dict.fromkeys(("santo domingo",), "DO"), + **dict.fromkeys(("san salvador",), "SV"), + **dict.fromkeys(("panama city",), "PA"), + **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("dubai", "abu dhabi"), "AE"), + **dict.fromkeys(("riyadh",), "SA"), + **dict.fromkeys(("doha",), "QA"), + **dict.fromkeys(("manama",), "BH"), + **dict.fromkeys(("salmiya",), "KW"), + **dict.fromkeys(("muscat",), "OM"), + **dict.fromkeys(("cairo", "maadi"), "EG"), + **dict.fromkeys(("casablanca",), "MA"), + **dict.fromkeys(("dakar",), "SN"), + **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("almaty",), "KZ"), + **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), + **dict.fromkeys(("dhaka",), "BD"), + **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", + "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"), **dict.fromkeys(("singapore",), "SG"), - **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), - **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), - **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), - **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), - **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), - **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), - **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), + **dict.fromkeys(("manila", "taguig"), "PH"), + **dict.fromkeys(("bangkok",), "TH"), + **dict.fromkeys(("phnom penh",), "KH"), + **dict.fromkeys(("taipei",), "TW"), + **dict.fromkeys(("tokyo", "osaka"), "JP"), + **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"), + **dict.fromkeys(("seoul",), "KR"), + **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", + "queensland"), "AU"), } -#: Keyed the way a place is folded before it is looked up, so a name can be -#: written as it is spelt. -_PLACES = {_fold(name): code for name, code in _PLACES.items()} +#: US states and Canadian provinces, by name and by the code boards write +#: after a comma ("Boston, MA", "Toronto, ON"). +_SUBDIVISIONS = """ +US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO +US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA +US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY +US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN +US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH +US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND +US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI +US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT +US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY +CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB +CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL +CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT +CA Nunavut NU +""" -#: The longest name first, so "new york city" wins over "york" and "northern -#: ireland" over "ireland". -_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) -#: US state codes after a comma, the way US boards write a city: "Boston, MA". -#: Read only when no name matched, since "DE" and "IN" are countries too. -_US_STATE = re.compile( - r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" - r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" - r"\b") +def _subdivisions() -> tuple[dict[str, str], dict[str, str]]: + names: dict[str, str] = {} + codes: dict[str, str] = {} + for line in _SUBDIVISIONS.split("\n"): + if line: + country, rest = line.split(" ", 1) + for entry in rest.split("|"): + name, code = entry.strip().rsplit(" ", 1) + names[_place_fold(name)] = country + codes[code] = country + return names, codes + + +_STATE_NAMES, _STATE_CODES = _subdivisions() + + +def _names() -> dict[str, tuple[str, str]]: + """Every folded name to `(code, kind)`; kind is `country`, `city` or + `state`. A name that is a city and a state ("New York") is the city, so + it does not beat another city in its line. A name that is a country and + a state ("Georgia") is the country here and read as the state only when + the line names a place in the US too.""" + names: dict[str, tuple[str, str]] = {} + for name, code in _STATE_NAMES.items(): + names[name] = (code, "state") + for name, code in _CITIES.items(): + names[_place_fold(name)] = (code, "city") + for line in _COUNTRY_NAMES.split("\n"): + if line: + code, rest = line.split(" ", 1) + for name in rest.split("|"): + names[_place_fold(name)] = (code, "country") + for name in _REGIONS: + names[name] = ("EU", "country") + return names + + +_NAMES = _names() + +#: A state name that is also a country's, read as the state only when the +#: line names a place in that state's country too. +_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES + if _NAMES[name][1] == "country"} + +#: The longest name in words, so a lookup tries "new york city" before "new +#: york" and "northern ireland" before "ireland". +_LONGEST = max(len(name.split()) for name in _NAMES) + +#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin". +_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b") + +#: A country code in parentheses, "Leicester (GB)", read only when no name matched. +_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)") + +#: Where one location in a line ends and the next begins. A state beats a +#: city only within one: "London; Ontario" is two places. +_SEGMENT = re.compile(r"[;|/\n]") + + +def _hits(folded: str) -> list[tuple[str, str, str]]: + """The names in a folded line, longest first at each word, as + `(name, code, kind)`.""" + words = folded.split() + hits = [] + i = 0 + while i < len(words): + for n in range(min(_LONGEST, len(words) - i), 0, -1): + name = " ".join(words[i:i + n]) + if name in _NAMES: + hits.append((name, *_NAMES[name])) + i += n + break + else: + i += 1 + return hits def country_of(place: str) -> str: - """The one country a board's location line names, as an ISO code, or '' - when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is - US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" - is EU, and "London, EMEA" is GB: a country beats the region it is in.""" - folded = f" {_fold(place or '')} " + """The one country a board's location line names, as an ISO code; `EU` + when it names several and all of them are in Europe; '' when it names + none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston, + MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or + New York" and "Remote, EU or US" are ''. + + Within one location (the line split at `;`, `|` and `/`) the widest unit + named wins: a country beats a state or a province, which beats a city. + So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A + country also beats the region it is in: "London, EMEA" is GB. + + A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", + count as the state only when the line names a place in the US too: + "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather + than Delaware. A country code in parentheses, "Leicester (GB)", is read + only when no name matched.""" + segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] + named = {code for _, hits in segments for _, code, _ in hits} found: set[str] = set() - for name in _PLACE_NAMES: - needle = f" {name} " - if needle in folded: - found.add(_PLACES[name]) - folded = folded.replace(needle, " ") - if not found and _US_STATE.search(place or ""): - # Only when nothing else named a country: "Berlin, DE" is Germany. - found.add("US") - if len(found) > 1: - found.discard("EU") - return found.pop() if len(found) == 1 else "" + for text, hits in segments: + shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named} + countries = {code for name, code, kind in hits + if kind == "country" and code != "EU" and name not in shared} + states = {code for _, code, kind in hits if kind == "state"} + states |= {_SHARED_NAMES[name] for name in shared} + states |= {country for code in _STATE_CODE.findall(text) + if (country := _STATE_CODES.get(code)) + and (code not in _ISO or country in named)} + found |= (countries or states or {code for _, code, _ in hits if code != "EU"}) + found |= {code for _, code, _ in hits if code == "EU"} + if not found: + found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO} + if len(found) == 1: + return found.pop() + if found and found <= _EUROPE_OR_EU: + countries = found - {"EU"} + return countries.pop() if "EU" in found and len(countries) == 1 else "EU" + return "" # --------------------------------------------------------------- vocabulary diff --git a/app/storage/schema.py b/app/storage/schema.py index 3219e87..5f6b2e7 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -17,7 +17,7 @@ import logging import sqlite3 from collections.abc import Callable -from app.models import posted_at, posted_host +from app.models import country_of, posted_at, posted_host log = logging.getLogger(__name__) @@ -381,6 +381,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; # its current rules is the whole point (see _recompute_function below). (11, "roles.function recomputed for every not-yet-extracted role, word-boundary " "classifier fix", lambda conn: _recompute_function(conn)), + (12, "roles.country recomputed from place for every not-yet-extracted role whose " + "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -429,6 +431,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None: len(updates), len(rows)) +def _recompute_country(conn: sqlite3.Connection) -> None: + """Migration 12's step. `app.models.country_of` used to read a US state + code over a country's ("Bogota, CO" was US), return '' for a line naming + several European countries, and miss most countries outside Europe; + `Store.backfill_from_place` fills only an empty column, so a wrong code + would stay. Recomputed from `place` for every role `extract` has not + read (its answer only ever filled an empty column, but a role it read is + left as it is) and whose board sent no `country` of its own in `raw` + (the `search` plugin's Jobindex and Adzuna rows do), which is what + `Store.upsert_role` would write for that role today.""" + rows = conn.execute( + "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall() + updates = [] + for uid, place, raw, old in rows: + try: + given = json.loads(raw or "{}").get("country") or "" + except (TypeError, ValueError, AttributeError): + given = "" + if given: + continue + new = country_of(place or "") + if new != old: + updates.append((new, uid)) + if updates: + conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates) + log.info("migration 12: recomputed country for %d of %d not-yet-extracted roles", + len(updates), len(rows)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index d95264f..d018147 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -42,7 +42,7 @@ Advanced + Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: [{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}] is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.
diff --git a/docs/decisions.md b/docs/decisions.md index af07529..41736ae 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -32,7 +32,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. -**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. +**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the location line to an ISO code from every ISO country by its English name (and its own-language name for a European one) and the cities the boards name most, where `extract` would be a paid call per role for the same answer. On the live data (2026-09-29) 7.0% of open roles resolve to nothing, down from 16.8%, and most of those write only "Hybrid", "Remote" or a region such as "Americas". The line is folded by `_place_fold`, which also maps ł, đ, ø and æ since NFKD leaves them whole; `_fold` stays as it is because it builds `company_key`. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a US state or a Canadian province, which beats a city, so "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A state code that is also a country's (DE, CO, MA, IN) counts as the state only when the line also names a US place, so "Bogota, CO" is CO and "Paderborn, DE" is ''. A line naming several countries is `EU` when all of them are European (the UN's Europe, plus Cyprus, Turkey and the Council of Europe's Caucasus members) and '' otherwise: "Dublin, London" is EU, "Remote, EU or US" is ''. `EU`, ISO's reserved code, also stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it; a country beats the region it is in ("London, EMEA" is GB). `backfill_from_place` only fills an empty column, so a fix that changes answers comes with a migration: v12 recomputes `country` for every role `extract` has not read and whose board sent no `country` in `raw`, which changed 1,705 open roles, 48 of them from one code to another or to '' (25 Hong Kong roles moved from CN to HK). **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. diff --git a/tests/test_search.py b/tests/test_search.py index a18f404..813bff4 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store: ("London or New York", ""), ("Remote", ""), ("", ""), + ("Wrocław", "PL"), # ł and đ do not decompose under NFKD + ("Łódź, Poland", "PL"), + ("Bogota, CO", "CO"), # a state code that is a country's too + ("Paderborn, DE", ""), # ... is no US state without a US place + ("Sliema, MT", ""), + ("Casablanca, MA", "MA"), + ("Cambridge, MA", "US"), + ("Chicago, IL", "US"), + ("Madrid, MD, Spain", "ES"), + ("Dublin, London", "EU"), # several European countries + ("Anywhere in France, Belgium, Spain", "EU"), + ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), + ("Remote, EU or US", ""), # a region and a country outside it + ("London, Ontario", "CA"), # a province beats the city before it + ("Bellevue, Washington", "US"), + ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code + ("Atlanta, Georgia", "US"), + ("Tbilisi, Georgia", "GE"), + ("Malaysia", "MY"), + ("Belgrade, Serbia", "RS"), + ("Dubai", "AE"), + ("United Arab Emirates", "AE"), + ("Buenos Aires, , Argentina", "AR"), + ("Leicester (GB)", "GB"), # a code in parentheses when nothing else + ("Cagliari (CA)", "IT"), # ... and not over a name + ("Hybrid", ""), + ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code diff --git a/tests/test_storage.py b/tests/test_storage.py index 0fb2a8a..0de97c4 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 11 + assert LATEST == 12 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -257,6 +257,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen +def test_v12_recomputes_country_from_place_unless_the_board_or_extract_set_it( + tmp_path: Path, clock: Clock) -> None: + path = tmp_path / "v11.db" + store = Store(path) + store.clock = clock + wrong = posting("1", place="Bogota, CO") # was read as US + several = posting("2", place="Dublin, London") # was '' + given = posting("3", place="Bogota, CO", country="US") # the board said so + extracted = posting("4", place="Bogota, CO") + for item in (wrong, several, given, extracted): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") + conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) + conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", + (extracted.uid,)) + conn.execute("PRAGMA user_version = 11") + + migrated = Store(path) + + assert migrated.migrations_applied == list(range(12, LATEST + 1)) + countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid) + if (role := migrated.role(uid)) is not None} + assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US", + extracted.uid: "US"} + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23" From 3abb44e10a9524a838e2900fa0de148c449961fc Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:12:14 +0200 Subject: [PATCH 2/4] Stream a board's body against MAX_BYTES and a 60 s deadline; read timeout 30 s _get streams the body with a running byte count, so an oversized board is refused at MAX_BYTES instead of after it is held whole, and bounds the whole request, redirects included, by DEADLINE, checked between chunks. The per-phase timeouts are httpx.Timeout(10, read=30), set from 84 live requests against mongodb, feedzai and the five Greenhouse boards with most open roles, with and without pay_transparency=true: the body arrives in under 1 s, the first byte waited up to 16.6 s, and the flag made no measurable difference. _paged works unchanged through it (tested over a mock transport, 429 included). Correct pull.py's BATCH_SIZE comment: a batch reads its boards in sequence, and the parallelism is the http pool's. Co-Authored-By: Claude Opus --- app/jobs/pull.py | 10 ++++--- docs/decisions.md | 2 ++ plugins/ats.py | 69 ++++++++++++++++++++++++++++++++++------------- tests/test_ats.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 22 deletions(-) diff --git a/app/jobs/pull.py b/app/jobs/pull.py index d0038b7..45aa04d 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -47,9 +47,13 @@ from app.storage import Store log = logging.getLogger(__name__) -#: As many boards in the air per job as there are `http` workers by default -#: (see `config.yaml`), so one pull keeps the pool busy without one job -#: holding all 190-odd boards' worth of retries against a single attempt. +#: Targets per `pull_batch` job. A batch reads its boards one after another, +#: so the parallelism is the number of batches the `http` pool runs at once +#: (8 workers by default, see `config.yaml`), not this number. On the live +#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough +#: to keep eight workers busy, while a batch that fails or is killed costs at +#: most ten targets' progress. A board's request is bounded by +#: `plugins.ats.DEADLINE` plus one read timeout. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" diff --git a/docs/decisions.md b/docs/decisions.md index 41736ae..cb4878a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -78,6 +78,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **The body cap is 96 MB, with one retry for Greenhouse.** openings.go's 8 MB cap was written against arbitrary URLs, while `ats` only fetches a fixed allow-list of seven ATS hosts, and the real run found `?content=true` boards between 9 and 41 MB. A board that still exceeds 96 MB retries once without `?content=true`, trading the description for the listing. +**A board request is streamed, with a 30 s read timeout and a 60 s deadline.** `_get` counts the body as it arrives and refuses it at `MAX_BYTES`, where reading `response.content` first held the whole body in memory before the check. httpx's timeouts are per phase, so a board that keeps sending a byte now and then never timed out; the deadline covers the whole request, redirects and body included, and is checked between chunks. openings.go's 10 s read timeout failed `greenhouse:mongodb` and `greenhouse:feedzai` in a pull of 2026-09-28. Measured on 2026-09-29, three runs each of those two and the five Greenhouse boards with the most open roles, with and without `pay_transparency=true` (84 requests, every one a CloudFront miss): a body of up to 10.4 MB arrives in under a second once it starts, and the first byte waited up to 16.6 s, with the three slowest at 16.6, 14.4 and 10.9 s. The flag made no measurable difference; the slow first byte is the origin's, with or without it. A read may wait 30 s, about twice the worst seen, and the deadline is 60 s, which leaves a large board room on a slower link. + **Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role. **Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse. diff --git a/plugins/ats.py b/plugins/ats.py index 334f386..842d86b 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -5,7 +5,7 @@ unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. -The field map, the redirect allow-list and the 10 s timeout are ported from +The field map and the redirect allow-list are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. Workable, SmartRecruiters, @@ -47,8 +47,7 @@ from desk.sources import ( log = logging.getLogger(__name__) -#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's -#: 8 MB body cap does not: it was written against arbitrary URLs, and every +#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every #: URL fetched here is `?content=true` against a fixed allow-list of seven ATS #: API hosts. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and @@ -56,9 +55,26 @@ log = logging.getLogger(__name__) #: to 96 MB so those boards fit; a board that still exceeds it gets one retry #: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least #: the titles land. -TIMEOUT = 10.0 MAX_BYTES = 96 << 20 +#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` +#: failed a pull of 2026-09-28 with a read timeout. Measured live on +#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse +#: boards with the most open roles (databricks, stripe, anthropic, feverup, +#: datadog), with and without `pay_transparency=true`, cached and with a +#: cache-busting parameter (84 requests; CloudFront answered every one +#: "Miss", so all reached the origin): the body arrives in under 1 s once it +#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the +#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, +#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. +#: The flag made no difference that three runs could tell apart from that +#: noise. So a read may wait 30 s, about twice the worst first byte seen, +#: and a whole request, redirects and body included, 60 s (see `_get`), +#: about four times the slowest seen, which still leaves a 96 MB body room +#: on a link far slower than the one measured. +TIMEOUTS = httpx.Timeout(10.0, read=30.0) +DEADLINE = 60.0 + #: host -> (feed URL template, careers URL template). `{slug}` is filled in; #: recruitee and personio put the slug in the hostname instead of the path. _BOARDS: dict[str, tuple[str, str]] = { @@ -141,29 +157,44 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, """Fetch one board feed: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none.""" + sends `payload` as JSON, `{}` when there is none. + + The body is streamed and counted as it arrives, so an oversized one is + refused at `MAX_BYTES` rather than after it has been read whole, and + the whole request, redirects included, must end within `DEADLINE`: the + per-read `TIMEOUTS` alone let a board that trickles bytes run on for + good. The deadline is checked between chunks, so it can be overrun by + at most one read timeout.""" allow = _allowed_host(host, slug) + deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") - response = client.request( - method, current, timeout=TIMEOUT, follow_redirects=False, + with client.stream( + method, current, timeout=TIMEOUTS, follow_redirects=False, headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, json=(payload or {}) if method == "POST" else None, - ) - if response.is_redirect: - location = response.headers.get("location") or "" - if not location: - raise ValueError("redirect with no location") - current = str(httpx.URL(current).join(location)) - continue - response.raise_for_status() - body = response.content - if len(body) > MAX_BYTES: - raise _TooLarge(f"response too large ({len(body)} bytes)") - return body + ) as response: + if response.is_redirect: + location = response.headers.get("location") or "" + if not location: + raise ValueError("redirect with no location") + current = str(httpx.URL(current).join(location)) + continue + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > MAX_BYTES: + raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") + if time.monotonic() > deadline: + raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", + request=response.request) + chunks.append(chunk) + return b"".join(chunks) raise ValueError("too many redirects") diff --git a/tests/test_ats.py b/tests/test_ats.py index a10f426..0823ee2 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -7,6 +7,7 @@ from __future__ import annotations import json import sys from pathlib import Path +from collections.abc import Callable, Iterator from typing import Any import httpx @@ -279,6 +280,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type] +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_get_stops_reading_at_max_bytes_rather_than_after_the_body( + monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[int] = [] + + def chunks() -> Iterator[bytes]: + for i in range(100): + sent.append(i) + yield b"x" * 10 + + monkeypatch.setattr(ats, "MAX_BYTES", 25) + with _client(lambda request: httpx.Response(200, content=chunks())) as client, \ + pytest.raises(ats._TooLarge): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + assert len(sent) == 3 # refused on the third chunk, not the hundredth + + +def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive( + monkeypatch: pytest.MonkeyPatch) -> None: + clock = iter(range(0, 1000, 10)) + monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock))) + monkeypatch.setattr(ats, "DEADLINE", 25.0) + body = (b"x" for _ in range(100)) + with _client(lambda request: httpx.Response(200, content=body)) as client, \ + pytest.raises(httpx.ReadTimeout, match="within 25 s"): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + + +def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v0/postings/old": + return httpx.Response(301, headers={"location": "/v0/postings/acme"}) + if request.url.path == "/v0/postings/away": + return httpx.Response(302, headers={"location": "https://evil.example/"}) + return httpx.Response(200, content=b"[]") + + with _client(handler) as client: + assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]" + with pytest.raises(ats._Refused): + ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme") + + +def test_paging_reads_through_the_streamed_get(no_delay: None) -> None: + """`_paged` over the real `_get`: the POST body carries the cursor, and a + 429 on a later page still reaches it as an `HTTPStatusError`.""" + bodies: list[Any] = [] + + def handler(request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content or b"{}") + bodies.append(payload) + if payload.get("token") == "stop": + return httpx.Response(429) + page = json.loads(_page("workable-2.json" if payload else "workable.json")) + if payload: + page["total"], page["nextPage"] = 4, "stop" + return httpx.Response(200, json=page) + + with _client(handler) as client: + rows, complete = ats._fetch_rows(client, "workable:exotec") + assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}] + assert (len(rows), complete) == (3, False) + + # ----------------------------------------------------------------- paging def _page(name: str) -> bytes: From 91d15fcc7d835d133f39b08e035b256747fbe8eb Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:16:23 +0200 Subject: [PATCH 3/4] A change made while score runs gets one follow-up score, and the page waits on it score is solo, so a request during a run was deduped into the running job, which had already read the old state. _score_after_change returns a queued score when one has not started, else enqueues a plain score and, if that dedupes to a running job, one follow-up with depends_on it, after="done" and dedup_key score:after:. POST /api/score answers with that job and agents.js waits on it; the affinity route answers with score_job. The flaky affinity test was pipeline's race, not the app's: Jobs.enqueue re-raises the UNIQUE IntegrityError when the live job it collided with finishes before the SELECT that should hand it back. The app retries. Co-Authored-By: Claude Opus --- app/static/agents.js | 8 ++++-- app/web/api.py | 65 +++++++++++++++++++++++++++++++++++++++----- docs/decisions.md | 2 +- tests/test_web.py | 47 +++++++++++++++++++++++++++++++- 4 files changed, 110 insertions(+), 12 deletions(-) diff --git a/app/static/agents.js b/app/static/agents.js index 9274b49..5b85be5 100644 --- a/app/static/agents.js +++ b/app/static/agents.js @@ -289,14 +289,16 @@ render(); } - // Start the `score` job and wait for it, so "scored" means the Roles tab - // already orders by what was saved. `said` is what happened first. + // Ask for a `score` and wait for the job the server answers with, so + // "scored" means the Roles tab already orders by what was saved. While a + // score is running that is a follow-up queued after it, since the running + // one read the agent before the save. `said` is what happened first. async function rescore(status, agent, said) { const runs = () => link("/settings/sources#runs", "follow it under Runs"); status.replaceChildren(`${said}. Scoring every open role; `, runs(), "."); let job; try { - job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } }); + job = await api("/api/score", { method: "POST" }); } catch (err) { status.textContent = `${said}, but scoring did not start.`; return; diff --git a/app/web/api.py b/app/web/api.py index 374ef7c..65915fd 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations +import sqlite3 from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Query, Response @@ -349,18 +350,68 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]: return {"closed": closed} +def _score_after_change(queue: Any) -> Job | None: + """The `score` job whose result will include a change just written. + + A `score` that is queued and has not started will read the change, so + it is the answer: the plain one, keyed `score`, or a follow-up. Else a + plain request, which the queue dedupes to a live job keyed `score` + because the kind is `solo`. When that job is already running it read + the old state, so it gets one follow-up that waits for it (`depends_on`, + `after="done"`), deduped on `score:after:`: ten requests during one + run are one extra pass. A pull's `score` stage is a plain `score` job + too, queued when its stage starts, so a request during a pull joins it. + + The retry is for a race in `pipeline.table.Jobs.enqueue`: when the + INSERT collides with a live job that finishes before the SELECT after + it, the SELECT finds nothing and the `IntegrityError` is re-raised. That + job has finished by then, so a second try succeeds.""" + for attempt in range(3): + try: + queued = queue.jobs.last("score", states=("queued",)) + if queued is not None and (queued.dedup_key == "score" + or queued.dedup_key.startswith("score:after:")): + return queued + job = queue.enqueue("score") + if job is not None and job.state == "running": + job = queue.enqueue("score", depends_on=[job.id], after="done", + dedup_key=f"score:after:{job.id}") + return job + except sqlite3.IntegrityError: + if attempt == 2: + raise + return None + + +@router.post("/score") +def request_score(queue: QueueDep) -> Job: + """Score every agent's roles after a change: now, or right after the + `score` that is already running (see `_score_after_change`). Returns the + job to wait for; the Search agents page waits on it after a save.""" + job = _score_after_change(queue) + if job is None: + raise HTTPException(409, "Scoring would not start.") + return job + + class AffinityBody(BaseModel): affinity: int #: Whose affinity; the default agent when absent. agent: int | None = None +class AffinityOut(Company): + #: The `score` job whose result will include this change; None when none + #: could be queued. + score_job: int | None = None + + @router.post("/companies/{key}/affinity") -def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company: +def set_affinity(key: str, body: AffinityBody, store: StoreDep, + queue: QueueDep) -> AffinityOut: """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the default agent. A change here is not seen in Best match until `score` has - run: the queue dedupes to a live job of that kind, so several clicks are - one run.""" + run, so it queues one (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") owner = _agent_or_default(store, body.agent) @@ -368,8 +419,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) company = store.set_affinity(owner.id, key, body.affinity) except KeyError: raise HTTPException(404, f"no such company: {key}") from None - queue.enqueue("score") - return company + job = _score_after_change(queue) + return AffinityOut(**company.model_dump(), score_job=job.id if job else None) # ----------------------------------------------------------------- sources @@ -589,8 +640,8 @@ class AgentBody(BaseModel): @router.post("/agents") def save_agent(body: AgentBody, store: StoreDep) -> Agent: - """Scores are not recomputed here: that is the `score` task, which the - page starts after a save.""" + """Scores are not recomputed here: the page asks `POST /api/score` after + a save and waits on the job it answers with.""" try: return store.save_agent(body.name, body.intent, body.gates, body.weights, body.enabled, agent_id=body.id, limits=body.limits) diff --git a/docs/decisions.md b/docs/decisions.md index cb4878a..052d6d6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -158,7 +158,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **"Hidden" is the word the UI uses for affinity -1; the code keeps `mute` and `hide_muted`.** Renaming the columns would touch every reader of `agents` and `companies` for no behavioural gain, so only the toggle's title, the filter label and the dimmed row say "hidden" in the UI. -**A company affinity change enqueues `score`.** `POST /api/companies/{key}/affinity` queues the `score` job right after the update, deduped to a live run of the same kind so ten clicks are one pass. Otherwise an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows at all until then, so they are missing from that order entirely. +**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. `pipeline`'s `Jobs.enqueue` has a race: when its INSERT collides with a live job that finishes before the SELECT after it, it re-raises the `IntegrityError` (`tests/test_web.py`'s affinity test failed on it about one run in ten). The app retries, which succeeds because that job has finished; the fix belongs upstream in `../boilerplate`. **A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair. diff --git a/tests/test_web.py b/tests/test_web.py index 5d2fb69..4835b99 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app.""" from __future__ import annotations import json +import sqlite3 from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -18,6 +19,7 @@ from app.config import Config, ServerConfig from app.main import create_app from app.storage import Store from app.web import MOVED, SETTINGS, TABS +from app.web.api import _score_after_change from app.web.pages import SECTIONS from auth import AuthConfig from desk.models import Item @@ -444,7 +446,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert res.json()["affinity"] == 2 assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr] assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr] - assert queue.jobs.last("score", states=("queued", "running", "done")) is not None + assert queue.jobs.get(res.json()["score_job"]).kind == "score" res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id}) assert res.status_code == 200 and res.json()["affinity"] == -1 @@ -458,6 +460,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert client.get("/api/companies", params={"agent": 999}).status_code == 404 +def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None: + """No worker runs here (no client), so the test moves the jobs itself.""" + first = _score_after_change(queue) + assert first is not None and first.dedup_key == "score" + assert _score_after_change(queue).id == first.id # queued: it will read the change + + assert queue.jobs.claim({"cpu": 1}).id == first.id + during = {_score_after_change(queue).id for _ in range(10)} + assert len(during) == 1 and first.id not in during + follow = queue.jobs.get(during.pop()) + assert (follow.depends_on, follow.after, follow.dedup_key) == ( + [first.id], "done", f"score:after:{first.id}") + + queue.jobs.finish(first.id, "done") + assert _score_after_change(queue).id == follow.id # queued, not started: still the one + + assert queue.jobs.claim({"cpu": 1}).id == follow.id + after = _score_after_change(queue) + assert after.id not in (first.id, follow.id) and after.dedup_key == "score" + + +def test_score_after_change_retries_the_queues_dedup_race( + queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: + """`Jobs.enqueue` re-raises the UNIQUE failure when the live job it + collided with finishes before it reads it back.""" + real, calls = queue.enqueue, [] + + def racy(*args: Any, **kwargs: Any) -> Any: + calls.append(args) + if len(calls) == 1: + raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key") + return real(*args, **kwargs) + + monkeypatch.setattr(queue, "enqueue", racy) + assert _score_after_change(queue) is not None and len(calls) == 2 + + +def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None: + res = client.post("/api/score") + assert res.status_code == 200 + assert queue.jobs.get(res.json()["id"]).kind == "score" + + # ----------------------------------------------------------- applications def test_save_creates_an_application_with_a_saved_event( From b2c8c08f7ff92758591f57d34caac251bf8cfeef Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 15:19:55 +0200 Subject: [PATCH 4/4] Judge an employer known only from a search; the Companies list says "from job boards" Store.set_affinity creates a minimal company row (key, and the name its roles give most often) for a key with roles and no row, so a Jobindex or Adzuna employer can be hidden or liked; the reader draws the control for any role with a company_key and reloads the company list after the first change. /api/companies derives from_search: no ats subscription (detached included) and no ats, careers_url or domain. companies.js labels it "from job boards" and keeps it out of "Not readable yet". Co-Authored-By: Claude Opus --- app/static/companies.js | 9 ++++++--- app/static/roles.js | 10 ++++++---- app/storage/store.py | 17 +++++++++++++++-- app/web/api.py | 17 +++++++++++++---- docs/decisions.md | 2 ++ tests/test_web.py | 38 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 78 insertions(+), 15 deletions(-) diff --git a/app/static/companies.js b/app/static/companies.js index c228f67..1476069 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -71,9 +71,12 @@ : "own site"; sub.appendChild(badge); sub.append(" · not readable yet"); + } else if (company.from_search) { + // Nobody followed it: its roles came from a search, and it was judged. + sub.append("from job boards"); } else { - // Every row comes from the seed, Follow or discover: none of them - // found where this company posts its roles. + // Followed by the seed, Follow or discover, and none of them found + // where this company posts its roles. sub.append("board not found"); } if (company.careers_url && !company.board) { @@ -214,7 +217,7 @@ case "hidden": return company.affinity === -1; case "on": return company.on === true; case "off": return company.on === false; - case "unreadable": return company.subscription_id == null; + case "unreadable": return company.subscription_id == null && !company.from_search; default: return true; } } diff --git a/app/static/roles.js b/app/static/roles.js index 8180ea9..5b7cef9 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -800,12 +800,14 @@ } /* The company's ★ ▲ ⊘, the one place a company is judged outside the - Companies page. A company seen only through a search has no row of its - own to judge. A change dims or undims its rows in the feed. */ + Companies page. A company seen only through a search has no row until + its first change creates one, so the list is reloaded then. A change + dims or undims its rows in the feed. */ function renderAffinity(slot, r) { const company = companies.get(r.company_key); - if (!company) { slot.replaceChildren(); return; } - slot.replaceChildren(affinityControl(r.company_key, company.affinity, () => { + if (!r.company_key) { slot.replaceChildren(); return; } + slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => { + if (!company) await loadCompanies(state.filter.agent).catch(() => {}); renderAffinity(slot, r); renderFeed(); }, state.filter.agent)); diff --git a/app/storage/store.py b/app/storage/store.py index a37b3b1..20c8d04 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1067,11 +1067,24 @@ class Store: """One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 favourite. 0 deletes the row, since no row means neutral. Returns the company with this agent's affinity. Raises ValueError for another - value and KeyError for an unknown company or agent.""" + value and KeyError for an unknown company or agent. + + A key with roles and no company row, an employer known only from a + search, gets a row here holding only its key and the name its roles + give it, the most frequent one: nothing else can be judged without + one. It has no board, `ats`, `careers_url` or `domain`, which is how + the Companies list tells it from a followed company.""" _check_affinity(affinity) with self.transaction() as conn: - self._known(conn, "companies", "key", key) self._known(conn, "agents", "id", agent_id) + if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None: + named = conn.execute( + """SELECT company FROM roles WHERE company_key=? AND company<>'' + GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC + LIMIT 1""", (key,)).fetchone() + if named is None: + raise KeyError(key) + self._insert_company(conn, key, named[0], {}) if affinity == 0: conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?", (agent_id, key)) diff --git a/app/web/api.py b/app/web/api.py index 65915fd..991cdaf 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -268,7 +268,10 @@ class CompanyOut(BaseModel): no `board` and an `ats` is one the app cannot read yet; `careers_url` is where a person can. `failing_since` and `failing_reason` say since when the board has failed on every fetch and why it last did, both empty - while it answers.""" + while it answers. `from_search` marks a row that exists only because a + search listed its roles and somebody judged it (`Store.set_affinity`): + no board, not even a detached one, and no `ats`, `careers_url` or + `domain`, which every way of following a company writes one of.""" key: str name: str @@ -283,6 +286,7 @@ class CompanyOut(BaseModel): country: str = "" failing_since: str = "" failing_reason: str = "" + from_search: bool = False def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -305,6 +309,7 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut """Every company, with `affinity` as `agent` (the default agent when absent) has it; 0 throughout when there is no agent.""" boards = _company_boards(store) + ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} counts = store.open_role_counts() failing = jobs.pull.failing_boards(store) out = [] @@ -316,7 +321,9 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut open_roles=counts.get(c.key, 0), ats=c.ats, ats_slug=c.ats_slug, careers_url=c.careers_url, country=c.country, - failing_since=since, failing_reason=reason)) + failing_since=since, failing_reason=reason, + from_search=not (c.key in ever_boarded or c.ats + or c.careers_url or c.domain))) return out @@ -410,8 +417,10 @@ class AffinityOut(Company): def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> AffinityOut: """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the - default agent. A change here is not seen in Best match until `score` has - run, so it queues one (`_score_after_change`) and answers with its id.""" + default agent. A key with roles and no company row gets one first (see + `Store.set_affinity`); 404 for a key with neither. A change here is not + seen in Best match until `score` has run, so it queues one + (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") owner = _agent_or_default(store, body.agent) diff --git a/docs/decisions.md b/docs/decisions.md index 052d6d6..a2a2f18 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -58,6 +58,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A company row carries `ats`, `ats_slug`, `country` and `city`.** `ats` is what discovery found whether or not the app reads it; a company with an `ats` and no board subscription is one it cannot read yet. `ats_slug` is the name that host knows the company by (the `acme` of `acme.teamtailor.com`), so the parser written later has its targets without a second discovery run; with a board it is the board's slug. `country` and `city` come from the seed for now and are the first of the company facts the feed will filter by; funding, size and industry come from a source that has them, not from a column filled by hand. +**An employer known only from a search gets a company row when it is first judged, and the Companies list derives that it is one.** Jobindex and Adzuna roles carry a `company_key` with no row in `companies` (67 keys with open roles on the live data of 2026-09-29), so they could not be hidden or liked. `Store.set_affinity` now creates a row for a key that has roles, named by the name its roles give most often, with nothing else set, and the reader draws the control for any role with a `company_key`. Such a row has no board, `ats` or `careers_url`, which the Companies list read as "board not found" and put under "Not readable yet". The rule: a company with no `ats` subscription (not even a detached one) and no `ats`, `careers_url` or `domain` is from job boards (`from_search` on `/api/companies`), labelled so and left out of that filter. Derived rather than a column, because every way of following a company writes one of those (the seed and discover a domain, Follow and `set-board` a board, which detaching hides but keeps), and a search-only company that later gets a domain or a board is followed from then on without a flag to flip. + **Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API. **A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. diff --git a/tests/test_web.py b/tests/test_web.py index 4835b99..196960e 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -389,7 +389,7 @@ def test_companies_lists_board_and_open_role_count( sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme", ats="greenhouse", country="DK") store.upsert_role(posting("1", company="Acme"), sub) - store.add_company("Other", key="other") # no board found + store.add_company("Other", key="other", domain="other.example") # no board found store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable careers_url="https://novo.example/careers") @@ -399,13 +399,47 @@ def test_companies_lists_board_and_open_role_count( assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, "ats": "greenhouse", "ats_slug": "", "careers_url": "", - "country": "DK", "failing_since": "", "failing_reason": ""} + "country": "DK", "failing_since": "", "failing_reason": "", + "from_search": False} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") [novo] = [c for c in res.json() if c["key"] == "novo"] assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == ( "", None, "successfactors", "https://novo.example/careers") + assert not any(c["from_search"] for c in res.json()) + + +def test_judging_a_company_known_only_from_a_search_creates_its_row( + client: TestClient, store: Store) -> None: + """Jobindex and Adzuna employers have roles and no company row. The first + affinity change makes a minimal one, named by its roles, which the + Companies list marks `from_search`, unlike a followed company without a + board, and a company whose board was detached.""" + store.save_agent("A") + search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") + for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")): + item = posting(f"n{i}", company=name) + item.source = "search" + store.upsert_role(item, search) + key = "novo-nordisk" + assert store.company(key) is None + + res = client.post(f"/api/companies/{key}/affinity", json={"affinity": -1}) + assert res.status_code == 200, res.text + assert (res.json()["name"], res.json()["affinity"]) == ("Novo Nordisk A/S", -1) + assert client.post("/api/companies/nobody/affinity", json={"affinity": 1}).status_code == 404 + + store.add_company("Pleo", key="pleo", domain="pleo.io") # followed, no board found + sub = store.add_subscription("ats", "lever:gone", company="Gone", board="lever") + store.add_company("Gone", key="gone") + store.set_company_board("gone", "") # detached + assert not store.subscription(sub.id).show # type: ignore[union-attr] + rows = {c["key"]: c for c in client.get("/api/companies").json()} + assert rows["gone"]["board"] == "" + assert rows[key]["from_search"] and rows[key]["open_roles"] == 3 + assert (rows[key]["board"], rows[key]["ats"], rows[key]["careers_url"]) == ("", "", "") + assert not rows["pleo"]["from_search"] and not rows["gone"]["from_search"] def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,