Merge main into wp-review-contract
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
fe2a1b0347
36 files changed
+1741
-596
No files matched your search
+152
-74
@@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch
|
||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||
the sweep of an oversized paged board across pulls, the 10% guard on a
|
||||
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
||||
the sweep of an oversized paged board across pulls, the 10% guard on every
|
||||
read that is not a sweep (`read`), the per-role detail reads
|
||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||
as they were.
|
||||
|
||||
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
||||
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
||||
as they were. Every reader declares `READS`; a module without it is refused
|
||||
by the package before it gets here.
|
||||
|
||||
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||
|
||||
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
||||
"seen", "rows", "whole"}`, continued by the next pull
|
||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
||||
the 10% guard
|
||||
count `{"asks", "count"}`, the roles the last read that was not a
|
||||
sweep listed, for the 10% guard
|
||||
failing `{"since", "reason"}`, kept by the Source (`record_failure`);
|
||||
the guard gives way once `since` is `Limits.stale_days` old
|
||||
remote_unasked "1" when the reach wanted remote and the read could not
|
||||
ask for it (`Listing.remote_unasked`), else ""
|
||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||
swept "1" when the last stamped listing finished a sweep carried over
|
||||
from an earlier pull (`Listing.swept`), else "". Offset paging
|
||||
@@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count
|
||||
#: How long a board's `facets` are kept before they are asked again.
|
||||
FACETS_TTL = timedelta(days=1)
|
||||
|
||||
#: A filtered or `empty="suspect"` read listing under this share of what the
|
||||
#: same read listed last time counts as a failure.
|
||||
#: A read listing under this share of what the same read listed last time
|
||||
#: counts as a failure (`read`).
|
||||
SUSPECT_SHARE = 0.1
|
||||
|
||||
#: The last count from which the guard holds every read. A read that is
|
||||
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||
#: a count of 1, as it was before every read was; a whole read of a board
|
||||
#: that listed a handful of roles can drop to none by itself.
|
||||
SUSPECT_MIN = 10
|
||||
|
||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
||||
|
||||
@@ -105,8 +113,8 @@ class TooLarge(ValueError):
|
||||
|
||||
|
||||
class Suspect(Exception):
|
||||
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
||||
of what it listed last time."""
|
||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
||||
(see `read`)."""
|
||||
|
||||
|
||||
class MissingKey(ValueError):
|
||||
@@ -115,12 +123,15 @@ class MissingKey(ValueError):
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Limits:
|
||||
"""What one Source allows a read: pages, the pause between requests, and
|
||||
the detail reads a pull."""
|
||||
"""What one Source allows a read: pages, the pause between requests, the
|
||||
detail reads a pull, and the days after which the 10% guard accepts the
|
||||
answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
|
||||
for a Source run outside a pull, never)."""
|
||||
|
||||
max_pages: int = MAX_PAGES
|
||||
page_delay: float = PAGE_DELAY
|
||||
detail_cap: int = DETAIL_CAP
|
||||
stale_days: int | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -134,7 +145,9 @@ class Listing:
|
||||
neither are the whole board. `swept` says a complete listing finished a
|
||||
sweep carried over from an earlier pull: `ids` were read across pulls,
|
||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||
roles back past the cursor, so `ids` can miss a role that is still open."""
|
||||
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||
as well and could not, so its `ids` say nothing of the remote roles."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -144,12 +157,14 @@ class Listing:
|
||||
asks: str = ""
|
||||
narrowed: bool = False
|
||||
swept: bool = False
|
||||
remote_unasked: bool = False
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
|
||||
def declared(module: ModuleType) -> bool:
|
||||
"""Whether `module` declares `READS`, and so takes `ask`."""
|
||||
"""Whether `module` declares `READS`. Every reader does, so the runner no
|
||||
longer asks; `app.jobs.reach` still does."""
|
||||
return isinstance(getattr(module, "READS", None), Reads)
|
||||
|
||||
|
||||
@@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
|
||||
reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
|
||||
"""The dimensions `reach` and then the params set, params winning, before
|
||||
they are narrowed to what the reader declares, and whether the read is
|
||||
wide. A reach of more than `MAX_SERIES` countries on a `one` country
|
||||
filter is dropped whole, which makes the read wide; a `remote` of False
|
||||
asks nothing."""
|
||||
given = reach or {}
|
||||
reached = _dimensions(module, given)
|
||||
wide = bool(given.get("wide"))
|
||||
if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||
reached = {}
|
||||
wide = True
|
||||
wanted = reached | _dimensions(module, dict(subscription_params or {}))
|
||||
if wanted.get("remote") is False:
|
||||
del wanted["remote"]
|
||||
return wanted, wide
|
||||
|
||||
|
||||
def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||
reach: Mapping[str, Any] | None = None) -> bool:
|
||||
"""Whether the reach or the params want remote roles, whether or not the
|
||||
reader can ask for them. `read_listing` compares it with what the series
|
||||
asked (`Listing.remote_unasked`)."""
|
||||
return _wanted(module, subscription_params, reach)[0].get("remote") is True
|
||||
|
||||
|
||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||
reach: Mapping[str, Any] | None = None, *,
|
||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||
@@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
||||
dropped. A `remote` of False asks nothing. A search's params go whole into
|
||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||
A module without `READS` gets `Ask()`.
|
||||
A remote the reader cannot filter on is dropped here; `remote_wanted`
|
||||
still says it was wanted.
|
||||
|
||||
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
||||
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
|
||||
@@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
||||
whole, since remote without its countries would read only the remote
|
||||
roles; over it with terms alone, the terms go too and the board is read
|
||||
unfiltered."""
|
||||
if not declared(module):
|
||||
return Ask()
|
||||
reads: Reads = module.READS
|
||||
params = dict(subscription_params or {})
|
||||
given = reach or {}
|
||||
reached = _dimensions(module, given)
|
||||
wide = bool(given.get("wide"))
|
||||
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||
reached = {}
|
||||
wide = True
|
||||
wanted = reached | _dimensions(module, params)
|
||||
if wanted.get("remote") is False:
|
||||
del wanted["remote"]
|
||||
wanted, wide = _wanted(module, params, given)
|
||||
declared_now = reads.filters.declared()
|
||||
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
||||
terms = _terms(given.get("terms"))
|
||||
@@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool:
|
||||
query. Such a read is never the whole board, nor the board within its
|
||||
places, so it closes nothing. A search's query is what
|
||||
the search is, and does not count."""
|
||||
board = not declared(module) or module.READS.scope == "board"
|
||||
board = module.READS.scope == "board"
|
||||
return bool(ask.terms) or (board and bool(ask.query))
|
||||
|
||||
|
||||
@@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask,
|
||||
remote part beside places is then left out. Each of those is then asked
|
||||
once per fetch term (`ask.terms`), with the term as `query`: the host
|
||||
intersects a keyword with its facets, as the survey found on every
|
||||
host, so the union by id is "one of the terms, in these places". `[ask]`
|
||||
for a module without `READS`."""
|
||||
if not declared(module):
|
||||
return [ask]
|
||||
host, so the union by id is "one of the terms, in these places"."""
|
||||
if ask.remote and (ask.countries or ask.places):
|
||||
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||
@@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]
|
||||
known = known_facets.get(dimension) or {}
|
||||
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
||||
if not mapped:
|
||||
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
||||
", ".join(values))
|
||||
log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
|
||||
dimension, ", ".join(values))
|
||||
return []
|
||||
if dimension == "country":
|
||||
countries = mapped
|
||||
@@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
for index, ask in enumerate(asks):
|
||||
if index:
|
||||
time.sleep(limits.page_delay)
|
||||
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
||||
url = module.feed_url(slug, ask)
|
||||
try:
|
||||
body = get(url)
|
||||
except TooLarge:
|
||||
@@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
||||
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
||||
can_sweep: bool = False,
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None,
|
||||
remote: bool = False) -> Listing:
|
||||
"""One target's listing, read and parsed. Raises on any failure of the
|
||||
first request, and on a reader that does not understand an answer.
|
||||
|
||||
@@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
its series and cursor; the sweep is complete when it reaches the end,
|
||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||
past it."""
|
||||
past it.
|
||||
|
||||
A filtered read that asks no series, because an `ids` dimension names
|
||||
no value the host's facets know, asked nothing, which is no evidence of
|
||||
anything: it is incomplete and closes nothing. `filtered` is a board's
|
||||
only: a search's places are what the search is, as its query is.
|
||||
`remote` says the reach or the params wanted remote roles; when no series
|
||||
asked for them (a reader without a remote filter, or one whose facets
|
||||
state none beside places) the listing is `remote_unasked`."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
reads: Reads = module.READS
|
||||
asks = series(module, ask, facets)
|
||||
is_filtered = filtered(ask)
|
||||
is_filtered = filtered(ask) and reads.scope == "board"
|
||||
by_words = narrowed(module, ask)
|
||||
unasked = (is_filtered and (remote or bool(ask.remote))
|
||||
and not any(a.remote or a.remote_ids for a in asks))
|
||||
key = fingerprint(asks)
|
||||
if not asks:
|
||||
return Listing([], True, [], is_filtered, None, key, by_words)
|
||||
log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
|
||||
"nothing asked, not closing", module.NAME, slug)
|
||||
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||
if not hasattr(module, "page"):
|
||||
rows = _whole(module, slug, get, asks, limits)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
|
||||
whole_rows = _whole(module, slug, get, asks, limits)
|
||||
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||
key, by_words, remote_unasked=unasked)
|
||||
|
||||
reads: Reads | None = module.READS if declared(module) else None
|
||||
never = reads is not None and reads.complete == "never"
|
||||
ceiling = reads.ceiling if reads is not None else None
|
||||
never = reads.complete == "never"
|
||||
ceiling = reads.ceiling
|
||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||
resumed = bool(sweep and sweep.get("asks") == key)
|
||||
if sweep and resumed:
|
||||
@@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
if pages:
|
||||
time.sleep(limits.page_delay)
|
||||
try:
|
||||
if reads is not None:
|
||||
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
||||
else:
|
||||
data, results, total, cursor = module.page(get, slug, cursor)
|
||||
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not pages or exc.response.status_code != 429:
|
||||
raise
|
||||
@@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||
if not stop:
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
||||
resumed and whole)
|
||||
resumed and whole, unasked)
|
||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||
module.NAME, slug, pages, stop)
|
||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
|
||||
remote_unasked=unasked)
|
||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||
"rows": listed, "whole": whole}
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
|
||||
remote_unasked=unasked)
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
@@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
||||
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
||||
(a country, a place, or remote),
|
||||
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
||||
if not declared(module) or not hasattr(module, "facets"):
|
||||
if not hasattr(module, "facets"):
|
||||
return {}
|
||||
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
||||
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
||||
@@ -659,46 +703,79 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
||||
|
||||
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
||||
slug: str, get: Get, ask: Ask | None = None, *,
|
||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
||||
limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
|
||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
||||
that is filtered, narrowed by terms, or whose reader declares
|
||||
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
|
||||
read listed last time (the same `fingerprint`, so a term read is only
|
||||
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
the rest once the rows are stored."""
|
||||
the facets it needs, the sweep it continues, and the 10% guard. Nothing
|
||||
is written here but the facets; `stamp` keeps the rest once the rows are
|
||||
stored.
|
||||
|
||||
The guard holds every read that is not a sweep: one that listed under
|
||||
`SUSPECT_SHARE` of what the same read listed last time (the same
|
||||
`fingerprint`, so a term read is only held to a term read with the same
|
||||
terms and places) raises `Suspect`, and the Source records the target
|
||||
as failing, so its roles stay open. A host whose answer changed shape
|
||||
parses to no rows with a 200, and closing a board on that would lose it
|
||||
whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
|
||||
that is filtered, narrowed or of an `empty="suspect"` reader. A board
|
||||
that really shrank is accepted once the target has been failing for
|
||||
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||
stamp then resets the count; a Recheck resets it at once
|
||||
(`plugins.ats.keep_size`)."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
||||
suspect = declared(module) and module.READS.empty == "suspect"
|
||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||
remote=remote)
|
||||
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
|
||||
if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
|
||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||
why = (f"listed {len(listing.ids)} roles, under "
|
||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||
if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||
module.NAME, slug, why, limits.stale_days)
|
||||
return listing
|
||||
raise Suspect(why)
|
||||
return listing
|
||||
|
||||
|
||||
def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
|
||||
"""Whether the failure kept under `key` began at least `days` ago; False
|
||||
for `days` None or no record."""
|
||||
since = str(_load(state, key).get("since") or "")
|
||||
if days is None or not since:
|
||||
return False
|
||||
try:
|
||||
began = datetime.fromisoformat(since)
|
||||
except ValueError:
|
||||
return False
|
||||
return datetime.now(UTC) - began >= timedelta(days=days)
|
||||
|
||||
|
||||
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
listing: Listing, ids: list[str]) -> None:
|
||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), whether it finished a sweep from earlier pulls (`swept`), the
|
||||
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||
whether a filtered read left remote unasked (`remote_unasked`), the
|
||||
sweep to continue or none, and the count for the 10% guard once a read
|
||||
is whole."""
|
||||
is not a sweep. A read that asked nothing keeps the count it had."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
else:
|
||||
state.pop(f"{prefix}:sweep:{target}", None)
|
||||
return
|
||||
state.pop(f"{prefix}:sweep:{target}", None)
|
||||
if listing.asks != fingerprint([]):
|
||||
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
||||
"count": len(listing.ids)})
|
||||
|
||||
@@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||
name, slug, row["source_id"], ", ".join(ignored))
|
||||
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
||||
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||
if taken.get("location") and "places" not in taken:
|
||||
taken["places"] = [{"text": taken["location"], "remote": None}]
|
||||
filled[key] = taken
|
||||
@@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
||||
for row in rows:
|
||||
key = compound(row)
|
||||
if key in filled:
|
||||
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
||||
out.append(row | {"description": ""} | filled[key])
|
||||
elif key in detailed:
|
||||
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
||||
out.append(row | {"description": "", "location": "", "places": [],
|
||||
"remote": None})
|
||||
else:
|
||||
out.append(row | {"description": ""}) # type: ignore[operator]
|
||||
out.append(row | {"description": ""})
|
||||
if listed is not None:
|
||||
done &= listed
|
||||
failing &= listed
|
||||
@@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
||||
"""A row's places: its `places`, or, for a row without (an undeclared
|
||||
reader's), its `location` as the one place, split nothing."""
|
||||
"""A row's places: its `places`, or, for a row without (a raw row stored
|
||||
before readers returned places), its `location` as the one place, split
|
||||
nothing."""
|
||||
places = row.get("places")
|
||||
if not isinstance(places, list):
|
||||
location = str(row.get("location") or "").strip()
|
||||
|
||||
Reference in new issue
Block a user