diff --git a/app/storage/store.py b/app/storage/store.py index 8ccd69e..a8dc590 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -415,12 +415,18 @@ class Store: columns = ", ".join(row) marks = ", ".join(f":{c}" for c in row) with self.transaction() as conn: - existed = conn.execute("SELECT 1 FROM roles WHERE uid=?", - (row["uid"],)).fetchone() is not None + before = conn.execute("SELECT description FROM roles WHERE uid=?", + (row["uid"],)).fetchone() conn.execute( f"INSERT INTO roles ({columns}) VALUES ({marks}) " f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row) - return not existed + # A role listed without its text is embedded from its title, and + # `embed` encodes a role once. When the text arrives on a later + # pull, as it does for a host that states it on the role's own + # page, the vector goes, so the next `embed` reads the text. + if before is not None and not before["description"] and row["description"]: + conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],)) + return before is None def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None: """One role with its tags, and with `vote`, `voted_at`, `score`, diff --git a/plugins/ats.py b/plugins/ats.py index 97992f8..dfe716e 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -618,7 +618,9 @@ DETAIL_CAP = 200 #: `location` (`place`, `place_key`, `country`), `remote` and `currency` are #: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only #: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the -#: earliest date a role ever had. `title` and `url` are not here, since the +#: earliest date a role ever had. A filled column is still overwritten by +#: another filled value, so `_read_details` withholds the listing's place and +#: remote for a role already read. `title` and `url` are not here, since the #: listing always states them and would overwrite them; nor are `department` #: (no column of its own, only `raw` and the classifier's guess) and #: `salary_text`, both rewritten from the listing on every pull. @@ -641,7 +643,10 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: is left out with a debug line. Every row not read in detail in this call goes out with an empty description: a role read on an earlier pull keeps the one it has, since `upsert_role` never overwrites a filled column with - an empty value, and a teaser from the listing must not replace it. + an empty value, and a teaser from the listing must not replace it. A role + read on an earlier pull goes out without the listing's place and remote + as well, for the same reason: the listing states them every pull + ("2 Locations"), and would replace what the role's own page said. A detail that answers 404 or 410 counts as read: the role is gone or has no page of its own, and asking again would not change that. One that @@ -691,7 +696,15 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} done.add(key) failing.discard(key) - out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows] + out = [] + for row in rows: + key = compound(row) + if key in filled: + out.append(row | {"description": ""} | filled[key]) + elif key in detailed: + out.append(row | {"description": "", "location": "", "remote": None}) + else: + out.append(row | {"description": ""}) if complete: listed = {compound(row) for row in rows} done &= listed diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py index f5661cd..1d694f3 100644 --- a/plugins/atshosts/__init__.py +++ b/plugins/atshosts/__init__.py @@ -113,6 +113,8 @@ listing's empty value; any other key is left out with a debug log line. on a pull is sent with an empty description, so the one read earlier stays. A 404 or a 410 counts as read; any other failure is logged and retried on a later pull; a 429 ends the detail reads of that board for the pull. +A role read in detail on an earlier pull is sent without the +listing's place and remote, so what its own page said about them stays. Completeness is decided by the listing alone. **robots.txt.** With `robots_page(slug)`, the shared code reads diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py index f4443c5..a5eb53c 100644 --- a/tests/test_atshosts.py +++ b/tests/test_atshosts.py @@ -328,6 +328,30 @@ def test_detail_is_read_for_unread_roles_only( assert state["ats:complete:fake:acme"] == "1" +def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """The listing states a place on every pull, and a filled value replaces + a filled column, so for a role already read it is withheld.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full", + f"{base}/3": RuntimeError("down")}) + module = _fake_host(detail=lambda get, slug, row: { + "description": get(f"{base}/{row['source_id']}").decode()}) + listing = module.parse + module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"} + for row in listing(body, slug)] + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"]), + "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} + items, _state = _run(monkeypatch, module, state) + + by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items} + assert by_id["fake:acme:1"] == ("", "", None) + # Read now, and its page stated neither: the listing's stand. + assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid") + # Never read: the listing's stand until its page is. + assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid") + + def test_detail_reads_stop_at_the_cap_and_the_rest_wait( monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: base = "https://jobs.fake.example/acme" diff --git a/tests/test_storage.py b/tests/test_storage.py index 066d927..0015b04 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -549,6 +549,24 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen( ("acme", "senior backend engineer", "berlin") +def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None: + listed = posting() + listed.body = "" + store.upsert_role(listed) + store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) + assert store.roles_to_embed("m", 10) == [] + + store.upsert_role(listed) # listed again, still without text + assert store.roles_to_embed("m", 10) == [] + store.upsert_role(posting()) # the role's own page was read + assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid] + + store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))]) + store.upsert_role(listed) # later pulls send no text + store.upsert_role(posting()) # or the same text again + assert store.roles_to_embed("m", 10) == [] + + def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None: store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000", salary_max=95000, currency="EUR", department="Eng"))