diff --git a/app/jobs/pull.py b/app/jobs/pull.py index d0038b7..45aa04d 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -47,9 +47,13 @@ from app.storage import Store log = logging.getLogger(__name__) -#: As many boards in the air per job as there are `http` workers by default -#: (see `config.yaml`), so one pull keeps the pool busy without one job -#: holding all 190-odd boards' worth of retries against a single attempt. +#: Targets per `pull_batch` job. A batch reads its boards one after another, +#: so the parallelism is the number of batches the `http` pool runs at once +#: (8 workers by default, see `config.yaml`), not this number. On the live +#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough +#: to keep eight workers busy, while a batch that fails or is killed costs at +#: most ten targets' progress. A board's request is bounded by +#: `plugins.ats.DEADLINE` plus one read timeout. BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" diff --git a/app/models.py b/app/models.py index 71b91ba..57579d1 100644 --- a/app/models.py +++ b/app/models.py @@ -66,121 +66,538 @@ def company_key(company: str) -> str: # ---------------------------------------------------------------- geography -#: Folded names a board writes in its location line, to an ISO code: every -#: European country in English and its own language, the cities the boards -#: name most, and the non-European places that fill the seed boards, so those -#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. -_PLACES: dict[str, str] = { - **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), - **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), - **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", - "ghent"), "BE"), - **dict.fromkeys(("bulgaria", "sofia"), "BG"), - **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), - **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), - **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), - # Folding keeps ø and æ, which are letters of their own rather than an o - # or an a with a mark, so København is listed beside Kobenhavn. - **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", - "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", - "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", - "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", - "birkerød"), "DK"), - **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), - **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), - **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", - "bordeaux", "lille"), "FR"), - **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", - "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", - "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", - "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), - **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), - **dict.fromkeys(("hungary", "budapest"), "HU"), - **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), - **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", - "torino", "bologna", "florence"), "IT"), - **dict.fromkeys(("latvia", "riga"), "LV"), - **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), - **dict.fromkeys(("luxembourg",), "LU"), - **dict.fromkeys(("malta", "valletta"), "MT"), - **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", - "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", +#: Letters NFKD leaves whole because they are letters of their own rather +#: than a base letter with a mark, spelt the way boards write them in ASCII. +_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe", + "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"}) + + +def _place_fold(text: str) -> str: + """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København" + meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is, + since it also builds `company_key`, the primary key of `companies`.""" + return _fold((text or "").casefold().translate(_LETTERS)) + + +#: Every ISO 3166-1 country by its English name, plus the names boards also +#: write for it: its own language's for a European one, and the usual short +#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one. +_COUNTRY_NAMES = """ +AD Andorra +AE United Arab Emirates | UAE | U.A.E. | Emirates +AF Afghanistan +AG Antigua and Barbuda +AI Anguilla +AL Albania | Shqipëria | Shqipëri +AM Armenia | Hayastan +AO Angola +AQ Antarctica +AR Argentina +AS American Samoa +AT Austria | Österreich +AU Australia +AW Aruba +AX Åland Islands | Åland +AZ Azerbaijan | Azərbaycan +BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina +BB Barbados +BD Bangladesh +BE Belgium | België | Belgique | Belgien +BF Burkina Faso +BG Bulgaria | България | Balgariya +BH Bahrain +BI Burundi +BJ Benin +BL Saint Barthélemy +BM Bermuda +BN Brunei | Brunei Darussalam +BO Bolivia +BQ Caribbean Netherlands | Bonaire +BR Brazil | Brasil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus | Беларусь | Belarus' +BZ Belize +CA Canada +CC Cocos Islands | Cocos (Keeling) Islands +CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa +CF Central African Republic +CG Republic of the Congo | Congo-Brazzaville +CH Switzerland | Schweiz | Suisse | Svizzera | Svizra +CI Côte d'Ivoire | Ivory Coast +CK Cook Islands +CL Chile +CM Cameroon +CN China | People's Republic of China +CO Colombia +CR Costa Rica +CU Cuba +CV Cabo Verde | Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus | Κύπρος | Kypros | Kibris +CZ Czechia | Czech Republic | Česko | Česká republika +DE Germany | Deutschland +DJ Djibouti +DK Denmark | Danmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia | Eesti +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain | España | Espanya +ET Ethiopia +FI Finland | Suomi +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands | Føroyar | Færøerne +FR France +GA Gabon +GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland +GD Grenada +GE Georgia | Sakartvelo +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece | Ελλάδα | Hellas | Ellada +GS South Georgia and the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island and McDonald Islands +HN Honduras +HR Croatia | Hrvatska +HT Haiti +HU Hungary | Magyarország +ID Indonesia +IE Ireland | Éire +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland | Ísland +IT Italy | Italia +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN Saint Kitts and Nevis +KP North Korea +KR South Korea | Korea | Republic of Korea +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic +LB Lebanon +LC Saint Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania | Lietuva +LU Luxembourg | Lëtzebuerg | Luxemburg +LV Latvia | Latvija +LY Libya +MA Morocco +MC Monaco +MD Moldova | Republic of Moldova +ME Montenegro | Crna Gora +MF Saint Martin +MG Madagascar +MH Marshall Islands +MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија +ML Mali +MM Myanmar | Burma +MN Mongolia +MO Macao | Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico | México +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands | The Netherlands | Nederland | Holland +NO Norway | Norge | Noreg +NP Nepal +NR Nauru +NU Niue +NZ New Zealand | Aotearoa +OM Oman +PA Panama | Panamá +PE Peru | Perú +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland | Polska +PM Saint Pierre and Miquelon +PN Pitcairn Islands +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania | România +RS Serbia | Srbija | Србија +RU Russia | Russian Federation | Россия | Rossiya +RW Rwanda +SA Saudi Arabia | KSA +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden | Sverige +SG Singapore +SH Saint Helena +SI Slovenia | Slovenija +SJ Svalbard and Jan Mayen | Svalbard +SK Slovakia | Slovensko +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST São Tomé and Príncipe +SV El Salvador +SX Sint Maarten +SY Syria +SZ Eswatini | Swaziland +TC Turks and Caicos Islands +TD Chad +TF French Southern Territories +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL Timor-Leste | East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey | Türkiye +TT Trinidad and Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine | Україна | Ukraina +UG Uganda +UM United States Minor Outlying Islands +US United States | United States of America | USA | U.S.A. | US | U.S. +UY Uruguay +UZ Uzbekistan +VA Vatican City | Holy See +VC Saint Vincent and the Grenadines +VE Venezuela +VG British Virgin Islands +VI U.S. Virgin Islands | US Virgin Islands +VN Vietnam | Viet Nam +VU Vanuatu +WF Wallis and Futuna +WS Samoa +XK Kosovo | Kosova +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe +""" + +#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK. +_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line) + +#: The countries `EU` stands for when a line names several: the UN +#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and +#: the Council of Europe's members in the Caucasus and Turkey, which +#: European employers list beside their other offices. +_EUROPE = frozenset(( + "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE", + "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE", + "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL", + "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA", + "XK", +)) +_EUROPE_OR_EU = _EUROPE | {"EU"} + +#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them. +_REGIONS = ("europe", "emea", "eu", "european union", "eea") + +#: The cities boards name most, and a few regions below a country. A US +#: state or a Canadian province in the same part of the line beats one of +#: these ("London, Ontario" is Canada); a country name does not lose to it. +_CITIES: dict[str, str] = { + **dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"), + **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"), + **dict.fromkeys(("sofia",), "BG"), + **dict.fromkeys(("zagreb",), "HR"), + **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), + **dict.fromkeys(("prague", "praha", "brno"), "CZ"), + **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", + "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", + "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", + "brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"), + **dict.fromkeys(("tallinn", "tartu"), "EE"), + **dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", + "lille", "rennes", "amiens"), "FR"), + **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", + "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", + "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", + "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", + "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", + "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", + "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"), + **dict.fromkeys(("athens", "thessaloniki"), "GR"), + **dict.fromkeys(("budapest",), "HU"), + **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), + **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", + "florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"), + **dict.fromkeys(("riga",), "LV"), + **dict.fromkeys(("vilnius", "kaunas"), "LT"), + **dict.fromkeys(("valletta",), "MT"), + **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), - **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", - "gdansk", "poznan", "lodz"), "PL"), - **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), - **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), - "RO"), - **dict.fromkeys(("slovakia", "bratislava"), "SK"), - **dict.fromkeys(("slovenia", "ljubljana"), "SI"), - **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", - "malaga", "bilbao"), "ES"), - **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", - "uppsala"), "SE"), - **dict.fromkeys(("iceland", "reykjavik"), "IS"), - **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), - **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", - "wales", "northern ireland", "london", "manchester", "edinburgh", - "glasgow", "bristol", "cambridge uk", "oxford", "leeds", - "birmingham uk", "belfast"), "GB"), - **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", - "basel", "lausanne", "bern", "zug"), "CH"), - **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", - "nyc", "san francisco", "seattle", "boston", "chicago", "austin", - "los angeles", "palo alto", "mountain view", "san mateo", - "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", - "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", - "houston", "phoenix", "tempe", "portland", "salt lake city", - "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", - "nashville", "minneapolis"), "US"), - **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", - "waterloo"), "CA"), - **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", - "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), + "PL"), + **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), + **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), + **dict.fromkeys(("belgrade", "beograd"), "RS"), + **dict.fromkeys(("bratislava",), "SK"), + **dict.fromkeys(("ljubljana",), "SI"), + **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", + "granada", "santiago de compostela"), "ES"), + **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), + **dict.fromkeys(("reykjavik",), "IS"), + **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), + **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", + "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", + "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", + "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", + "slough", "southampton", "swindon", "liverpool", "redhill", + "milton keynes"), "GB"), + **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", + "sankt gallen", "st gallen"), "CH"), + **dict.fromkeys(("kyiv", "kiev"), "UA"), + **dict.fromkeys(("istanbul", "ankara"), "TR"), + **dict.fromkeys(("tirana",), "AL"), + **dict.fromkeys(("yerevan",), "AM"), + **dict.fromkeys(("tbilisi",), "GE"), + **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", + "boston", "chicago", "austin", "los angeles", "palo alto", + "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", + "menlo park", "redwood city", "costa mesa", "denver", "atlanta", + "washington dc", "washington d c", "miami", "dallas", "houston", + "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", + "philadelphia", "detroit", "san diego", "raleigh", "nashville", + "minneapolis", "cambridge ma", "bay area", "arlington"), "US"), + **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), + "CA"), + **dict.fromkeys(("buenos aires",), "AR"), + **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"), + **dict.fromkeys(("santiago", "santiago de chile"), "CL"), + **dict.fromkeys(("bogota",), "CO"), + **dict.fromkeys(("lima", "miraflores"), "PE"), + **dict.fromkeys(("montevideo",), "UY"), + **dict.fromkeys(("managua",), "NI"), + **dict.fromkeys(("santo domingo",), "DO"), + **dict.fromkeys(("san salvador",), "SV"), + **dict.fromkeys(("panama city",), "PA"), + **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("dubai", "abu dhabi"), "AE"), + **dict.fromkeys(("riyadh",), "SA"), + **dict.fromkeys(("doha",), "QA"), + **dict.fromkeys(("manama",), "BH"), + **dict.fromkeys(("salmiya",), "KW"), + **dict.fromkeys(("muscat",), "OM"), + **dict.fromkeys(("cairo", "maadi"), "EG"), + **dict.fromkeys(("casablanca",), "MA"), + **dict.fromkeys(("dakar",), "SN"), + **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("almaty",), "KZ"), + **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), + **dict.fromkeys(("dhaka",), "BD"), + **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", + "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"), **dict.fromkeys(("singapore",), "SG"), - **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), - **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), - **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), - **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), - **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), - **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), - **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), + **dict.fromkeys(("manila", "taguig"), "PH"), + **dict.fromkeys(("bangkok",), "TH"), + **dict.fromkeys(("phnom penh",), "KH"), + **dict.fromkeys(("taipei",), "TW"), + **dict.fromkeys(("tokyo", "osaka"), "JP"), + **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"), + **dict.fromkeys(("seoul",), "KR"), + **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", + "queensland"), "AU"), } -#: Keyed the way a place is folded before it is looked up, so a name can be -#: written as it is spelt. -_PLACES = {_fold(name): code for name, code in _PLACES.items()} +#: US states and Canadian provinces, by name and by the code boards write +#: after a comma ("Boston, MA", "Toronto, ON"). +_SUBDIVISIONS = """ +US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO +US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA +US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY +US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN +US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH +US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND +US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI +US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT +US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY +CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB +CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL +CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT +CA Nunavut NU +""" -#: The longest name first, so "new york city" wins over "york" and "northern -#: ireland" over "ireland". -_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) -#: US state codes after a comma, the way US boards write a city: "Boston, MA". -#: Read only when no name matched, since "DE" and "IN" are countries too. -_US_STATE = re.compile( - r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" - r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" - r"\b") +def _subdivisions() -> tuple[dict[str, str], dict[str, str]]: + names: dict[str, str] = {} + codes: dict[str, str] = {} + for line in _SUBDIVISIONS.split("\n"): + if line: + country, rest = line.split(" ", 1) + for entry in rest.split("|"): + name, code = entry.strip().rsplit(" ", 1) + names[_place_fold(name)] = country + codes[code] = country + return names, codes + + +_STATE_NAMES, _STATE_CODES = _subdivisions() + + +def _names() -> dict[str, tuple[str, str]]: + """Every folded name to `(code, kind)`; kind is `country`, `city` or + `state`. A name that is a city and a state ("New York") is the city, so + it does not beat another city in its line. A name that is a country and + a state ("Georgia") is the country here and read as the state only when + the line names a place in the US too.""" + names: dict[str, tuple[str, str]] = {} + for name, code in _STATE_NAMES.items(): + names[name] = (code, "state") + for name, code in _CITIES.items(): + names[_place_fold(name)] = (code, "city") + for line in _COUNTRY_NAMES.split("\n"): + if line: + code, rest = line.split(" ", 1) + for name in rest.split("|"): + names[_place_fold(name)] = (code, "country") + for name in _REGIONS: + names[name] = ("EU", "country") + return names + + +_NAMES = _names() + +#: A state name that is also a country's, read as the state only when the +#: line names a place in that state's country too. +_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES + if _NAMES[name][1] == "country"} + +#: The longest name in words, so a lookup tries "new york city" before "new +#: york" and "northern ireland" before "ireland". +_LONGEST = max(len(name.split()) for name in _NAMES) + +#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin". +_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b") + +#: A country code in parentheses, "Leicester (GB)", read only when no name matched. +_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)") + +#: Where one location in a line ends and the next begins. A state beats a +#: city only within one: "London; Ontario" is two places. +_SEGMENT = re.compile(r"[;|/\n]") + + +def _hits(folded: str) -> list[tuple[str, str, str]]: + """The names in a folded line, longest first at each word, as + `(name, code, kind)`.""" + words = folded.split() + hits = [] + i = 0 + while i < len(words): + for n in range(min(_LONGEST, len(words) - i), 0, -1): + name = " ".join(words[i:i + n]) + if name in _NAMES: + hits.append((name, *_NAMES[name])) + i += n + break + else: + i += 1 + return hits def country_of(place: str) -> str: - """The one country a board's location line names, as an ISO code, or '' - when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is - US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" - is EU, and "London, EMEA" is GB: a country beats the region it is in.""" - folded = f" {_fold(place or '')} " + """The one country a board's location line names, as an ISO code; `EU` + when it names several and all of them are in Europe; '' when it names + none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston, + MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or + New York" and "Remote, EU or US" are ''. + + Within one location (the line split at `;`, `|` and `/`) the widest unit + named wins: a country beats a state or a province, which beats a city. + So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A + country also beats the region it is in: "London, EMEA" is GB. + + A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", + count as the state only when the line names a place in the US too: + "Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather + than Delaware. A country code in parentheses, "Leicester (GB)", is read + only when no name matched.""" + segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] + named = {code for _, hits in segments for _, code, _ in hits} found: set[str] = set() - for name in _PLACE_NAMES: - needle = f" {name} " - if needle in folded: - found.add(_PLACES[name]) - folded = folded.replace(needle, " ") - if not found and _US_STATE.search(place or ""): - # Only when nothing else named a country: "Berlin, DE" is Germany. - found.add("US") - if len(found) > 1: - found.discard("EU") - return found.pop() if len(found) == 1 else "" + for text, hits in segments: + shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named} + countries = {code for name, code, kind in hits + if kind == "country" and code != "EU" and name not in shared} + states = {code for _, code, kind in hits if kind == "state"} + states |= {_SHARED_NAMES[name] for name in shared} + states |= {country for code in _STATE_CODE.findall(text) + if (country := _STATE_CODES.get(code)) + and (code not in _ISO or country in named)} + found |= (countries or states or {code for _, code, _ in hits if code != "EU"}) + found |= {code for _, code, _ in hits if code == "EU"} + if not found: + found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO} + if len(found) == 1: + return found.pop() + if found and found <= _EUROPE_OR_EU: + countries = found - {"EU"} + return countries.pop() if "EU" in found and len(countries) == 1 else "EU" + return "" # --------------------------------------------------------------- vocabulary diff --git a/app/static/agents.js b/app/static/agents.js index bace965..2736e03 100644 --- a/app/static/agents.js +++ b/app/static/agents.js @@ -289,14 +289,16 @@ render(); } - // Start the `score` job and wait for it, so "scored" means the Roles tab - // already orders by what was saved. `said` is what happened first. + // Ask for a `score` and wait for the job the server answers with, so + // "scored" means the Roles tab already orders by what was saved. While a + // score is running that is a follow-up queued after it, since the running + // one read the agent before the save. `said` is what happened first. async function rescore(status, agent, said) { const runs = () => link("/settings/sources#runs", "follow it under Runs"); status.replaceChildren(`${said}. Scoring every open role; `, runs(), "."); let job; try { - job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } }); + job = await api("/api/score", { method: "POST" }); } catch (err) { status.textContent = `${said}, but scoring did not start.`; return; diff --git a/app/static/companies.js b/app/static/companies.js index c228f67..1476069 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -71,9 +71,12 @@ : "own site"; sub.appendChild(badge); sub.append(" · not readable yet"); + } else if (company.from_search) { + // Nobody followed it: its roles came from a search, and it was judged. + sub.append("from job boards"); } else { - // Every row comes from the seed, Follow or discover: none of them - // found where this company posts its roles. + // Followed by the seed, Follow or discover, and none of them found + // where this company posts its roles. sub.append("board not found"); } if (company.careers_url && !company.board) { @@ -214,7 +217,7 @@ case "hidden": return company.affinity === -1; case "on": return company.on === true; case "off": return company.on === false; - case "unreadable": return company.subscription_id == null; + case "unreadable": return company.subscription_id == null && !company.from_search; default: return true; } } diff --git a/app/static/roles.js b/app/static/roles.js index e3a92bf..80b03c1 100644 --- a/app/static/roles.js +++ b/app/static/roles.js @@ -834,12 +834,14 @@ } /* The company's ★ ▲ ⊘, the one place a company is judged outside the - Companies page. A company seen only through a search has no row of its - own to judge. A change dims or undims its rows in the feed. */ + Companies page. A company seen only through a search has no row until + its first change creates one, so the list is reloaded then. A change + dims or undims its rows in the feed. */ function renderAffinity(slot, r) { const company = companies.get(r.company_key); - if (!company) { slot.replaceChildren(); return; } - slot.replaceChildren(affinityControl(r.company_key, company.affinity, () => { + if (!r.company_key) { slot.replaceChildren(); return; } + slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => { + if (!company) await loadCompanies(state.filter.agent).catch(() => {}); renderAffinity(slot, r); renderFeed(); }, state.filter.agent)); diff --git a/app/storage/schema.py b/app/storage/schema.py index 0d599ec..10d4185 100644 --- a/app/storage/schema.py +++ b/app/storage/schema.py @@ -17,7 +17,7 @@ import logging import sqlite3 from collections.abc import Callable -from app.models import posted_at, posted_host +from app.models import country_of, posted_at, posted_host log = logging.getLogger(__name__) @@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT ''; "could not be normalised into salary_min/salary_max/currency", """ ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT ''; """), + (13, "roles.country recomputed from place for every not-yet-extracted role whose " + "board gave none, country lookup fix", lambda conn: _recompute_country(conn)), ) #: What a file created from `SCHEMA` today is already at. @@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None: len(updates), len(rows)) +def _recompute_country(conn: sqlite3.Connection) -> None: + """Migration 13's step. `app.models.country_of` used to read a US state + code over a country's ("Bogota, CO" was US), return '' for a line naming + several European countries, and miss most countries outside Europe; + `Store.backfill_from_place` fills only an empty column, so a wrong code + would stay. Recomputed from `place` for every role `extract` has not + read (its answer only ever filled an empty column, but a role it read is + left as it is) and whose board sent no `country` of its own in `raw` + (the `search` plugin's Jobindex and Adzuna rows do), which is what + `Store.upsert_role` would write for that role today.""" + rows = conn.execute( + "SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall() + updates = [] + for uid, place, raw, old in rows: + try: + given = json.loads(raw or "{}").get("country") or "" + except (TypeError, ValueError, AttributeError): + given = "" + if given: + continue + new = country_of(place or "") + if new != old: + updates.append((new, uid)) + if updates: + conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates) + log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles", + len(updates), len(rows)) + + def user_version(conn: sqlite3.Connection) -> int: return int(conn.execute("PRAGMA user_version").fetchone()[0]) diff --git a/app/storage/store.py b/app/storage/store.py index da4e42b..29ee2d0 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1094,11 +1094,24 @@ class Store: """One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2 favourite. 0 deletes the row, since no row means neutral. Returns the company with this agent's affinity. Raises ValueError for another - value and KeyError for an unknown company or agent.""" + value and KeyError for an unknown company or agent. + + A key with roles and no company row, an employer known only from a + search, gets a row here holding only its key and the name its roles + give it, the most frequent one: nothing else can be judged without + one. It has no board, `ats`, `careers_url` or `domain`, which is how + the Companies list tells it from a followed company.""" _check_affinity(affinity) with self.transaction() as conn: - self._known(conn, "companies", "key", key) self._known(conn, "agents", "id", agent_id) + if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None: + named = conn.execute( + """SELECT company FROM roles WHERE company_key=? AND company<>'' + GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC + LIMIT 1""", (key,)).fetchone() + if named is None: + raise KeyError(key) + self._insert_company(conn, key, named[0], {}) if affinity == 0: conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?", (agent_id, key)) diff --git a/app/templates/settings/agents.html b/app/templates/settings/agents.html index 9d8ac08..229b3e2 100644 --- a/app/templates/settings/agents.html +++ b/app/templates/settings/agents.html @@ -42,7 +42,7 @@ Advanced + Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: [{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}] is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.
diff --git a/app/web/api.py b/app/web/api.py index 172094a..fa6ef07 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations +import sqlite3 from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Query, Response @@ -270,7 +271,10 @@ class CompanyOut(BaseModel): no `board` and an `ats` is one the app cannot read yet; `careers_url` is where a person can. `failing_since` and `failing_reason` say since when the board has failed on every fetch and why it last did, both empty - while it answers.""" + while it answers. `from_search` marks a row that exists only because a + search listed its roles and somebody judged it (`Store.set_affinity`): + no board, not even a detached one, and no `ats`, `careers_url` or + `domain`, which every way of following a company writes one of.""" key: str name: str @@ -285,6 +289,7 @@ class CompanyOut(BaseModel): country: str = "" failing_since: str = "" failing_reason: str = "" + from_search: bool = False def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -307,6 +312,7 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut """Every company, with `affinity` as `agent` (the default agent when absent) has it; 0 throughout when there is no agent.""" boards = _company_boards(store) + ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} counts = store.open_role_counts() failing = jobs.pull.failing_boards(store) out = [] @@ -318,7 +324,9 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut open_roles=counts.get(c.key, 0), ats=c.ats, ats_slug=c.ats_slug, careers_url=c.careers_url, country=c.country, - failing_since=since, failing_reason=reason)) + failing_since=since, failing_reason=reason, + from_search=not (c.key in ever_boarded or c.ats + or c.careers_url or c.domain))) return out @@ -352,18 +360,70 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]: return {"closed": closed} +def _score_after_change(queue: Any) -> Job | None: + """The `score` job whose result will include a change just written. + + A `score` that is queued and has not started will read the change, so + it is the answer: the plain one, keyed `score`, or a follow-up. Else a + plain request, which the queue dedupes to a live job keyed `score` + because the kind is `solo`. When that job is already running it read + the old state, so it gets one follow-up that waits for it (`depends_on`, + `after="done"`), deduped on `score:after:`: ten requests during one + run are one extra pass. A pull's `score` stage is a plain `score` job + too, queued when its stage starts, so a request during a pull joins it. + + The retry is for a race in `pipeline.table.Jobs.enqueue`: when the + INSERT collides with a live job that finishes before the SELECT after + it, the SELECT finds nothing and the `IntegrityError` is re-raised. That + job has finished by then, so a second try succeeds.""" + for attempt in range(3): + try: + queued = queue.jobs.last("score", states=("queued",)) + if queued is not None and (queued.dedup_key == "score" + or queued.dedup_key.startswith("score:after:")): + return queued + job = queue.enqueue("score") + if job is not None and job.state == "running": + job = queue.enqueue("score", depends_on=[job.id], after="done", + dedup_key=f"score:after:{job.id}") + return job + except sqlite3.IntegrityError: + if attempt == 2: + raise + return None + + +@router.post("/score") +def request_score(queue: QueueDep) -> Job: + """Score every agent's roles after a change: now, or right after the + `score` that is already running (see `_score_after_change`). Returns the + job to wait for; the Search agents page waits on it after a save.""" + job = _score_after_change(queue) + if job is None: + raise HTTPException(409, "Scoring would not start.") + return job + + class AffinityBody(BaseModel): affinity: int #: Whose affinity; the default agent when absent. agent: int | None = None +class AffinityOut(Company): + #: The `score` job whose result will include this change; None when none + #: could be queued. + score_job: int | None = None + + @router.post("/companies/{key}/affinity") -def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company: +def set_affinity(key: str, body: AffinityBody, store: StoreDep, + queue: QueueDep) -> AffinityOut: """-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the - default agent. A change here is not seen in Best match until `score` has - run: the queue dedupes to a live job of that kind, so several clicks are - one run.""" + default agent. A key with roles and no company row gets one first (see + `Store.set_affinity`); 404 for a key with neither. A change here is not + seen in Best match until `score` has run, so it queues one + (`_score_after_change`) and answers with its id.""" if body.affinity not in AFFINITY: raise HTTPException(422, "affinity is -1, 0, 1 or 2") owner = _agent_or_default(store, body.agent) @@ -371,8 +431,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) company = store.set_affinity(owner.id, key, body.affinity) except KeyError: raise HTTPException(404, f"no such company: {key}") from None - queue.enqueue("score") - return company + job = _score_after_change(queue) + return AffinityOut(**company.model_dump(), score_job=job.id if job else None) # ----------------------------------------------------------------- sources @@ -592,8 +652,8 @@ class AgentBody(BaseModel): @router.post("/agents") def save_agent(body: AgentBody, store: StoreDep) -> Agent: - """Scores are not recomputed here: that is the `score` task, which the - page starts after a save.""" + """Scores are not recomputed here: the page asks `POST /api/score` after + a save and waits on the job it answers with.""" try: return store.save_agent(body.name, body.intent, body.gates, body.weights, body.enabled, agent_id=body.id, limits=body.limits) diff --git a/docs/decisions.md b/docs/decisions.md index bc04bec..e8bf342 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -34,7 +34,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. -**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. +**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the location line to an ISO code from every ISO country by its English name (and its own-language name for a European one) and the cities the boards name most, where `extract` would be a paid call per role for the same answer. On the live data (2026-09-29) 7.0% of open roles resolve to nothing, down from 16.8%, and most of those write only "Hybrid", "Remote" or a region such as "Americas". The line is folded by `_place_fold`, which also maps ł, đ, ø and æ since NFKD leaves them whole; `_fold` stays as it is because it builds `company_key`. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a US state or a Canadian province, which beats a city, so "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A state code that is also a country's (DE, CO, MA, IN) counts as the state only when the line also names a US place, so "Bogota, CO" is CO and "Paderborn, DE" is ''. A line naming several countries is `EU` when all of them are European (the UN's Europe, plus Cyprus, Turkey and the Council of Europe's Caucasus members) and '' otherwise: "Dublin, London" is EU, "Remote, EU or US" is ''. `EU`, ISO's reserved code, also stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it; a country beats the region it is in ("London, EMEA" is GB). `backfill_from_place` only fills an empty column, so a fix that changes answers comes with a migration: v12 recomputes `country` for every role `extract` has not read and whose board sent no `country` in `raw`, which changed 1,705 open roles, 48 of them from one code to another or to '' (25 Hong Kong roles moved from CN to HK). **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. @@ -60,6 +60,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **A company row carries `ats`, `ats_slug`, `country` and `city`.** `ats` is what discovery found whether or not the app reads it; a company with an `ats` and no board subscription is one it cannot read yet. `ats_slug` is the name that host knows the company by (the `acme` of `acme.teamtailor.com`), so the parser written later has its targets without a second discovery run; with a board it is the board's slug. `country` and `city` come from the seed for now and are the first of the company facts the feed will filter by; funding, size and industry come from a source that has them, not from a column filled by hand. +**An employer known only from a search gets a company row when it is first judged, and the Companies list derives that it is one.** Jobindex and Adzuna roles carry a `company_key` with no row in `companies` (67 keys with open roles on the live data of 2026-09-29), so they could not be hidden or liked. `Store.set_affinity` now creates a row for a key that has roles, named by the name its roles give most often, with nothing else set, and the reader draws the control for any role with a `company_key`. Such a row has no board, `ats` or `careers_url`, which the Companies list read as "board not found" and put under "Not readable yet". The rule: a company with no `ats` subscription (not even a detached one) and no `ats`, `careers_url` or `domain` is from job boards (`from_search` on `/api/companies`), labelled so and left out of that filter. Derived rather than a column, because every way of following a company writes one of those (the seed and discover a domain, Follow and `set-board` a board, which detaching hides but keeps), and a search-only company that later gets a domain or a board is followed from then on without a flag to flip. + **Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API. **A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. @@ -80,6 +82,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **The body cap is 96 MB, with one retry for Greenhouse.** openings.go's 8 MB cap was written against arbitrary URLs, while `ats` only fetches a fixed allow-list of seven ATS hosts, and the real run found `?content=true` boards between 9 and 41 MB. A board that still exceeds 96 MB retries once without `?content=true`, trading the description for the listing. +**A board request is streamed, with a 30 s read timeout and a 60 s deadline.** `_get` counts the body as it arrives and refuses it at `MAX_BYTES`, where reading `response.content` first held the whole body in memory before the check. httpx's timeouts are per phase, so a board that keeps sending a byte now and then never timed out; the deadline covers the whole request, redirects and body included, and is checked between chunks. openings.go's 10 s read timeout failed `greenhouse:mongodb` and `greenhouse:feedzai` in a pull of 2026-09-28. Measured on 2026-09-29, three runs each of those two and the five Greenhouse boards with the most open roles, with and without `pay_transparency=true` (84 requests, every one a CloudFront miss): a body of up to 10.4 MB arrives in under a second once it starts, and the first byte waited up to 16.6 s, with the three slowest at 16.6, 14.4 and 10.9 s. The flag made no measurable difference; the slow first byte is the origin's, with or without it. A read may wait 30 s, about twice the worst seen, and the deadline is 60 s, which leaves a large board room on a slower link. + **Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role. **Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse. @@ -164,7 +168,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **"Hidden" is the word the UI uses for affinity -1; the code keeps `mute` and `hide_muted`.** Renaming the columns would touch every reader of `agents` and `companies` for no behavioural gain, so only the toggle's title, the filter label and the dimmed row say "hidden" in the UI. -**A company affinity change enqueues `score`.** `POST /api/companies/{key}/affinity` queues the `score` job right after the update, deduped to a live run of the same kind so ten clicks are one pass. Otherwise an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows at all until then, so they are missing from that order entirely. +**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. `pipeline`'s `Jobs.enqueue` has a race: when its INSERT collides with a live job that finishes before the SELECT after it, it re-raises the `IntegrityError` (`tests/test_web.py`'s affinity test failed on it about one run in ten). The app retries, which succeeds because that job has finished; the fix belongs upstream in `../boilerplate`. **A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair. diff --git a/plugins/ats.py b/plugins/ats.py index 6720d92..1c6acb0 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -5,7 +5,7 @@ unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. -The field map, the redirect allow-list and the 10 s timeout are ported from +The field map and the redirect allow-list are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. Workable, SmartRecruiters, @@ -47,8 +47,7 @@ from desk.sources import ( log = logging.getLogger(__name__) -#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's -#: 8 MB body cap does not: it was written against arbitrary URLs, and every +#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every #: URL fetched here is `?content=true` against a fixed allow-list of seven ATS #: API hosts. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and @@ -56,9 +55,26 @@ log = logging.getLogger(__name__) #: to 96 MB so those boards fit; a board that still exceeds it gets one retry #: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least #: the titles land. -TIMEOUT = 10.0 MAX_BYTES = 96 << 20 +#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` +#: failed a pull of 2026-09-28 with a read timeout. Measured live on +#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse +#: boards with the most open roles (databricks, stripe, anthropic, feverup, +#: datadog), with and without `pay_transparency=true`, cached and with a +#: cache-busting parameter (84 requests; CloudFront answered every one +#: "Miss", so all reached the origin): the body arrives in under 1 s once it +#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the +#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, +#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. +#: The flag made no difference that three runs could tell apart from that +#: noise. So a read may wait 30 s, about twice the worst first byte seen, +#: and a whole request, redirects and body included, 60 s (see `_get`), +#: about four times the slowest seen, which still leaves a 96 MB body room +#: on a link far slower than the one measured. +TIMEOUTS = httpx.Timeout(10.0, read=30.0) +DEADLINE = 60.0 + #: host -> (feed URL template, careers URL template). `{slug}` is filled in; #: recruitee and personio put the slug in the hostname instead of the path. _BOARDS: dict[str, tuple[str, str]] = { @@ -141,29 +157,44 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, """Fetch one board feed: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST - sends `payload` as JSON, `{}` when there is none.""" + sends `payload` as JSON, `{}` when there is none. + + The body is streamed and counted as it arrives, so an oversized one is + refused at `MAX_BYTES` rather than after it has been read whole, and + the whole request, redirects included, must end within `DEADLINE`: the + per-read `TIMEOUTS` alone let a board that trickles bytes run on for + good. The deadline is checked between chunks, so it can be overrun by + at most one read timeout.""" allow = _allowed_host(host, slug) + deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") - response = client.request( - method, current, timeout=TIMEOUT, follow_redirects=False, + with client.stream( + method, current, timeout=TIMEOUTS, follow_redirects=False, headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, json=(payload or {}) if method == "POST" else None, - ) - if response.is_redirect: - location = response.headers.get("location") or "" - if not location: - raise ValueError("redirect with no location") - current = str(httpx.URL(current).join(location)) - continue - response.raise_for_status() - body = response.content - if len(body) > MAX_BYTES: - raise _TooLarge(f"response too large ({len(body)} bytes)") - return body + ) as response: + if response.is_redirect: + location = response.headers.get("location") or "" + if not location: + raise ValueError("redirect with no location") + current = str(httpx.URL(current).join(location)) + continue + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > MAX_BYTES: + raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") + if time.monotonic() > deadline: + raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", + request=response.request) + chunks.append(chunk) + return b"".join(chunks) raise ValueError("too many redirects") diff --git a/tests/test_ats.py b/tests/test_ats.py index 3e7b12d..2574d56 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -7,6 +7,7 @@ from __future__ import annotations import json import sys from pathlib import Path +from collections.abc import Callable, Iterator from typing import Any import httpx @@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type] +def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_get_stops_reading_at_max_bytes_rather_than_after_the_body( + monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[int] = [] + + def chunks() -> Iterator[bytes]: + for i in range(100): + sent.append(i) + yield b"x" * 10 + + monkeypatch.setattr(ats, "MAX_BYTES", 25) + with _client(lambda request: httpx.Response(200, content=chunks())) as client, \ + pytest.raises(ats._TooLarge): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + assert len(sent) == 3 # refused on the third chunk, not the hundredth + + +def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive( + monkeypatch: pytest.MonkeyPatch) -> None: + clock = iter(range(0, 1000, 10)) + monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock))) + monkeypatch.setattr(ats, "DEADLINE", 25.0) + body = (b"x" for _ in range(100)) + with _client(lambda request: httpx.Response(200, content=body)) as client, \ + pytest.raises(httpx.ReadTimeout, match="within 25 s"): + ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme") + + +def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v0/postings/old": + return httpx.Response(301, headers={"location": "/v0/postings/acme"}) + if request.url.path == "/v0/postings/away": + return httpx.Response(302, headers={"location": "https://evil.example/"}) + return httpx.Response(200, content=b"[]") + + with _client(handler) as client: + assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]" + with pytest.raises(ats._Refused): + ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme") + + +def test_paging_reads_through_the_streamed_get(no_delay: None) -> None: + """`_paged` over the real `_get`: the POST body carries the cursor, and a + 429 on a later page still reaches it as an `HTTPStatusError`.""" + bodies: list[Any] = [] + + def handler(request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content or b"{}") + bodies.append(payload) + if payload.get("token") == "stop": + return httpx.Response(429) + page = json.loads(_page("workable-2.json" if payload else "workable.json")) + if payload: + page["total"], page["nextPage"] = 4, "stop" + return httpx.Response(200, json=page) + + with _client(handler) as client: + rows, complete = ats._fetch_rows(client, "workable:exotec") + assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}] + assert (len(rows), complete) == (3, False) + + # ----------------------------------------------------------------- paging def _page(name: str) -> bytes: diff --git a/tests/test_search.py b/tests/test_search.py index a18f404..813bff4 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store: ("London or New York", ""), ("Remote", ""), ("", ""), + ("Wrocław", "PL"), # ł and đ do not decompose under NFKD + ("Łódź, Poland", "PL"), + ("Bogota, CO", "CO"), # a state code that is a country's too + ("Paderborn, DE", ""), # ... is no US state without a US place + ("Sliema, MT", ""), + ("Casablanca, MA", "MA"), + ("Cambridge, MA", "US"), + ("Chicago, IL", "US"), + ("Madrid, MD, Spain", "ES"), + ("Dublin, London", "EU"), # several European countries + ("Anywhere in France, Belgium, Spain", "EU"), + ("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"), + ("Remote, EU or US", ""), # a region and a country outside it + ("London, Ontario", "CA"), # a province beats the city before it + ("Bellevue, Washington", "US"), + ("Tilburg, NB, Netherlands", "NL"), # a country beats a province code + ("Atlanta, Georgia", "US"), + ("Tbilisi, Georgia", "GE"), + ("Malaysia", "MY"), + ("Belgrade, Serbia", "RS"), + ("Dubai", "AE"), + ("United Arab Emirates", "AE"), + ("Buenos Aires, , Argentina", "AR"), + ("Leicester (GB)", "GB"), # a code in parentheses when nothing else + ("Cagliari (CA)", "IT"), # ... and not over a name + ("Hybrid", ""), + ("Distributed", ""), ]) def test_country_of(place: str, code: str) -> None: assert country_of(place) == code diff --git a/tests/test_storage.py b/tests/test_storage.py index 4ff99ee..96308c4 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None: "application_events", "agents", "scores", "state"} <= tables assert "idx_roles_cluster" in indexes assert store.migrations_applied == [] - assert LATEST == 12 + assert LATEST == 13 # A second open finds it current and applies nothing. assert Store(path).migrations_applied == [] @@ -260,6 +260,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses( assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen +def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it( + tmp_path: Path, clock: Clock) -> None: + path = tmp_path / "v11.db" + store = Store(path) + store.clock = clock + wrong = posting("1", place="Bogota, CO") # was read as US + several = posting("2", place="Dublin, London") # was '' + given = posting("3", place="Bogota, CO", country="US") # the board said so + extracted = posting("4", place="Bogota, CO") + for item in (wrong, several, given, extracted): + store.upsert_role(item) + with sqlite3.connect(path) as conn: + conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'") + conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,)) + conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?", + (extracted.uid,)) + conn.execute("PRAGMA user_version = 12") + + migrated = Store(path) + + assert migrated.migrations_applied == list(range(13, LATEST + 1)) + countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid) + if (role := migrated.role(uid)) is not None} + assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US", + extracted.uid: "US"} + + def test_state_is_a_mapping(store: Store) -> None: store.state["ats:last"] = "2026-09-23" assert store.state["ats:last"] == "2026-09-23" diff --git a/tests/test_web.py b/tests/test_web.py index f55ed6f..2aa4d35 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app.""" from __future__ import annotations import json +import sqlite3 from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -18,6 +19,7 @@ from app.config import Config, ServerConfig from app.main import create_app from app.storage import Store from app.web import MOVED, SETTINGS, TABS +from app.web.api import _score_after_change from app.web.pages import SECTIONS from auth import AuthConfig from desk.models import Item @@ -411,7 +413,7 @@ def test_companies_lists_board_and_open_role_count( sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") store.add_company("Acme", key="acme", ats="greenhouse", country="DK") store.upsert_role(posting("1", company="Acme"), sub) - store.add_company("Other", key="other") # no board found + store.add_company("Other", key="other", domain="other.example") # no board found store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable careers_url="https://novo.example/careers") @@ -421,13 +423,47 @@ def test_companies_lists_board_and_open_role_count( assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, "ats": "greenhouse", "ats_slug": "", "careers_url": "", - "country": "DK", "failing_since": "", "failing_reason": ""} + "country": "DK", "failing_since": "", "failing_reason": "", + "from_search": False} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") [novo] = [c for c in res.json() if c["key"] == "novo"] assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == ( "", None, "successfactors", "https://novo.example/careers") + assert not any(c["from_search"] for c in res.json()) + + +def test_judging_a_company_known_only_from_a_search_creates_its_row( + client: TestClient, store: Store) -> None: + """Jobindex and Adzuna employers have roles and no company row. The first + affinity change makes a minimal one, named by its roles, which the + Companies list marks `from_search`, unlike a followed company without a + board, and a company whose board was detached.""" + store.save_agent("A") + search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex") + for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")): + item = posting(f"n{i}", company=name) + item.source = "search" + store.upsert_role(item, search) + key = "novo-nordisk" + assert store.company(key) is None + + res = client.post(f"/api/companies/{key}/affinity", json={"affinity": -1}) + assert res.status_code == 200, res.text + assert (res.json()["name"], res.json()["affinity"]) == ("Novo Nordisk A/S", -1) + assert client.post("/api/companies/nobody/affinity", json={"affinity": 1}).status_code == 404 + + store.add_company("Pleo", key="pleo", domain="pleo.io") # followed, no board found + sub = store.add_subscription("ats", "lever:gone", company="Gone", board="lever") + store.add_company("Gone", key="gone") + store.set_company_board("gone", "") # detached + assert not store.subscription(sub.id).show # type: ignore[union-attr] + rows = {c["key"]: c for c in client.get("/api/companies").json()} + assert rows["gone"]["board"] == "" + assert rows[key]["from_search"] and rows[key]["open_roles"] == 3 + assert (rows[key]["board"], rows[key]["ats"], rows[key]["careers_url"]) == ("", "", "") + assert not rows["pleo"]["from_search"] and not rows["gone"]["from_search"] def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store, @@ -468,7 +504,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert res.json()["affinity"] == 2 assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr] assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr] - assert queue.jobs.last("score", states=("queued", "running", "done")) is not None + assert queue.jobs.get(res.json()["score_job"]).kind == "score" res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id}) assert res.status_code == 200 and res.json()["affinity"] == -1 @@ -482,6 +518,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score( assert client.get("/api/companies", params={"agent": 999}).status_code == 404 +def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None: + """No worker runs here (no client), so the test moves the jobs itself.""" + first = _score_after_change(queue) + assert first is not None and first.dedup_key == "score" + assert _score_after_change(queue).id == first.id # queued: it will read the change + + assert queue.jobs.claim({"cpu": 1}).id == first.id + during = {_score_after_change(queue).id for _ in range(10)} + assert len(during) == 1 and first.id not in during + follow = queue.jobs.get(during.pop()) + assert (follow.depends_on, follow.after, follow.dedup_key) == ( + [first.id], "done", f"score:after:{first.id}") + + queue.jobs.finish(first.id, "done") + assert _score_after_change(queue).id == follow.id # queued, not started: still the one + + assert queue.jobs.claim({"cpu": 1}).id == follow.id + after = _score_after_change(queue) + assert after.id not in (first.id, follow.id) and after.dedup_key == "score" + + +def test_score_after_change_retries_the_queues_dedup_race( + queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: + """`Jobs.enqueue` re-raises the UNIQUE failure when the live job it + collided with finishes before it reads it back.""" + real, calls = queue.enqueue, [] + + def racy(*args: Any, **kwargs: Any) -> Any: + calls.append(args) + if len(calls) == 1: + raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key") + return real(*args, **kwargs) + + monkeypatch.setattr(queue, "enqueue", racy) + assert _score_after_change(queue) is not None and len(calls) == 2 + + +def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None: + res = client.post("/api/score") + assert res.status_code == 200 + assert queue.jobs.get(res.json()["id"]).kind == "score" + + # ----------------------------------------------------------- applications def test_save_creates_an_application_with_a_saved_event(