From 5832146c93230158979273ab10768287ac6e04a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:17:00 +0100 Subject: [PATCH 1/3] Check each line a listing page adds against the data before it reaches the desk A watch on a page that lists many events keeps one event per line, so its diff is already the list of finds. check-conference-update now hands those lines to desk_leads.py listing, which drops a line whose edition the data tracks (or, for a line without a year, whose series it tracks) and queues the rest as candidates with the flag listing. The worklist routes them like any other candidate: a missing edition of a tracked series to the new-year-checker, a new series to the scouter. Quoted names on a line, as on the PSF trademark list, are each matched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGjzRsX9nEb4swsKYms --- .claude/skills/scouter/SKILL.md | 2 +- .github/workflows/check-conference-update.yml | 41 +++++-- tests/test_check_conference_update.py | 42 ++++--- tests/test_desk_leads.py | 67 +++++++++++ utils/desk_leads.py | 110 +++++++++++++++++- utils/desk_queue.py | 9 +- 6 files changed, 239 insertions(+), 32 deletions(-) diff --git a/.claude/skills/scouter/SKILL.md b/.claude/skills/scouter/SKILL.md index 0329574529..fd76bfa9dc 100644 --- a/.claude/skills/scouter/SKILL.md +++ b/.claude/skills/scouter/SKILL.md @@ -14,7 +14,7 @@ The desk hands you the daily `scout` item, or an item about a series no data fil - `lead-only`: an aggregator lead. Copy no value from it; take every value from a primary source. - `no-website`: the shared sheet has this edition but no website. Search for one. - `denylist-review: : `: the domain is on the denylist's `review` list. Copy the flag into the `flags` of the find's change set; the critic returns needs-maintainer. - - A `registry` item whose payload `registry` is `listing`: a watched page that lists many events changed. Each event name or link the payload's `diff` adds is a find, with the item's `url` as where it came from; fetch that page when the diff is cut short or names an event without a link. A listing page is a lead, never the evidence (step 4). + - `listing`: one line a watched page that lists many events added, such as "PyData Amsterdam". The item's `url` is that page and its payload `names` the names on the line. Find the event on the page or by search. A listing page is a lead, never the evidence (step 4). - A `maintainer-request` item: the series and URLs the comment names are the find, and the request settles its scope (step 2). - The `scout` item: search each of these, with `` standing for this year and next (the year of the worklist's `today`, which the desk gives with the item). - Targeted web searches: "PyCon ", "PyData ", "Python conference call for proposals", "PyDay", "Python Pizza", framework and scientific library names with "conference", and the same in other languages ("conferencia Python", "Python Konferenz"). diff --git a/.github/workflows/check-conference-update.yml b/.github/workflows/check-conference-update.yml index bdf417ef63..8f57fc8707 100644 --- a/.github/workflows/check-conference-update.yml +++ b/.github/workflows/check-conference-update.yml @@ -6,9 +6,11 @@ name: Check Conference Update # # conference-change comes from a watch on one conference's pages and queues a # signal for that series. listing-change comes from a watch on a page that -# lists many events (pydata.org/upcoming-events, barcamps.eu and the like) and -# queues a registry item, which the worklist always gives the scouter: the -# names and links the diff adds are the finds. +# lists many events (pydata.org/upcoming-events, conference.scipy.org, the +# PSF PyCon trademark list), filtered to one event per line. Each added line +# is checked against the data in code (utils/desk_leads.py listing): a tracked +# edition is dropped, and the rest are queued as candidates, which the +# worklist routes to the new-year-checker or the scouter. on: # Triggered by changedetection.io webhook @@ -101,16 +103,26 @@ jobs: name, year = clean_watch_title(title) name = " ".join(name.split()) # one line, for the step output - payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger} if listing: - payload["registry"] = "listing" + # One event per added line: the next step checks each against + # the data. Files, not step outputs, carry the page's text. + temp = os.environ["RUNNER_TEMP"] + with open(f"{temp}/listing-lines.txt", "w", encoding="utf-8") as f: + f.write(diff if diff.endswith("\n") or not diff else diff + "\n") + with open(f"{temp}/listing-url.txt", "w", encoding="utf-8") as f: + f.write(" ".join(resolve_watch_url(url.strip()).split())) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"series={name}\nlisting=true\n") + print(f"Checking {len(diff.splitlines())} added line(s) from {name or title}") + sys.exit(0) + payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger} if event_name == "workflow_dispatch": # The run id gives each manual request its own queue id, so # asking again with the same inputs is not taken as handled. payload["requested"] = os.environ["GITHUB_RUN_ID"] record = { "source": "changedetection", - "kind": "registry" if listing else "signal", + "kind": "signal", "title": title.strip(), # A watch that follows a series sends its URL with the year template in it "url": resolve_watch_url(url.strip()), @@ -121,9 +133,24 @@ jobs: f.write(json.dumps(record, ensure_ascii=False) + "\n") with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: out.write(f"series={name}\n") - print(f"Queueing a {trigger} {record['kind']} item for {name or title}") + print(f"Queueing a {trigger} signal for {name or title}") BUILD + - name: Setup Pixi + if: steps.record.outputs.listing == 'true' + uses: prefix-dev/setup-pixi@v0.11.2 + + # Each added line is one event. A line whose edition (or, without a + # year, whose series) the data tracks is dropped; the rest become + # candidates that the worklist routes like any other. + - name: Check the listing lines against the data + if: steps.record.outputs.listing == 'true' + run: | + pixi run python utils/desk_leads.py listing \ + --lines "$RUNNER_TEMP/listing-lines.txt" \ + --url "$(cat "$RUNNER_TEMP/listing-url.txt")" \ + --out "$RUNNER_TEMP/desk-records.jsonl" + - name: Write to the desk queue id: queue uses: ./.github/actions/desk-queue-write diff --git a/tests/test_check_conference_update.py b/tests/test_check_conference_update.py index 2d87993e33..0cef3c2694 100644 --- a/tests/test_check_conference_update.py +++ b/tests/test_check_conference_update.py @@ -43,8 +43,15 @@ def run(action, payload): monkeypatch.setenv("RECORDS", str(records)) monkeypatch.setenv("GITHUB_OUTPUT", str(output)) monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) - runpy.run_path(str(script), run_name="__main__") - return json.loads(records.read_text(encoding="utf-8")), output.read_text(encoding="utf-8") + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + code = 0 + try: + runpy.run_path(str(script), run_name="__main__") + except SystemExit as exit_: + code = exit_.code or 0 + assert code == 0 + record = json.loads(records.read_text(encoding="utf-8")) if records.exists() else None + return record, output.read_text(encoding="utf-8") return run @@ -67,27 +74,28 @@ def test_conference_change_queues_a_signal_with_the_fetched_url(self, dispatch): assert "registry" not in record["payload"] assert "series=PyCon Ireland" in output - def test_listing_change_queues_a_registry_item(self, dispatch): + def test_listing_change_hands_its_lines_to_the_data_check(self, dispatch, tmp_path): listing = { "url": "https://pydata.org/upcoming-events/", "title": "PyData Upcoming", "watch_uuid": "abc", - "diff": "+ PyData Lagos 2027", - } - record, _ = dispatch("listing-change", listing) - assert record["source"] == "changedetection" - assert record["kind"] == "registry" - assert record["payload"] == { - "diff": "+ PyData Lagos 2027", - "watch_uuid": "abc", - "trigger": "website_change", - "registry": "listing", + "diff": "PyData Lagos 2027\nPyData Amsterdam", } + record, output = dispatch("listing-change", listing) + assert record is None # the next step writes the records, one per line it keeps + assert "listing=true" in output.splitlines() + assert (tmp_path / "listing-lines.txt").read_text(encoding="utf-8") == "PyData Lagos 2027\nPyData Amsterdam\n" + assert (tmp_path / "listing-url.txt").read_text(encoding="utf-8") == "https://pydata.org/upcoming-events/" + + def test_a_signal_is_a_valid_queue_record(self, dispatch): + record, output = dispatch("conference-change", PAYLOAD) + assert normalise_record(record)["kind"] == "signal" + assert "listing=true" not in output - @pytest.mark.parametrize("action", ["conference-change", "listing-change"]) - def test_the_queue_accepts_both(self, dispatch, action): - record, _ = dispatch(action, PAYLOAD) - assert normalise_record(record)["kind"] == record["kind"] + def test_the_listing_steps_run_only_for_a_listing(self): + steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["queue"]["steps"] + guarded = [step["name"] for step in steps if step.get("if") == "steps.record.outputs.listing == 'true'"] + assert guarded == ["Setup Pixi", "Check the listing lines against the data"] def test_both_dispatch_types_trigger_the_workflow(self): # PyYAML reads the bare key `on` as True diff --git a/tests/test_desk_leads.py b/tests/test_desk_leads.py index b2b189584f..a26d435ceb 100644 --- a/tests/test_desk_leads.py +++ b/tests/test_desk_leads.py @@ -897,6 +897,73 @@ def pretalx_web(web): not_found(web, "pycongr27", *OTHER_PROBES) +LISTING_URL = "https://pydata.org/upcoming-events/" +LISTING_LINES = [ + "(added) SciPy India 2026", + "SciPy India 2027", + "PyData Lagos 2027", + "", + "+ PyData Berlin", + "PyData Lagos 2027", + '"Kiwi PyCon", "PyCon NZ" in New Zealand', + "“PyCon NP”, “PyCon Nepal” in Nepal", + "Brand New Conference", +] + + +class TestListing: + def test_names_are_the_quoted_ones_or_the_line(self): + assert desk_leads.listing_names('"PyCon AT", "PyCon Austria" in Austria') == ["PyCon AT", "PyCon Austria"] + assert desk_leads.listing_names("“PyCon NP”, “PyCon Nepal” in Nepal") == ["PyCon NP", "PyCon Nepal"] + assert desk_leads.listing_names(" PyData Amsterdam ") == ["PyData Amsterdam"] + + def test_drops_what_the_data_tracks_and_routes_the_rest(self, index): + records = desk_leads.listing_records(LISTING_LINES, LISTING_URL, index) + assert [(r["title"], r["year"], r["payload"]["classification"]) for r in records] == [ + ("SciPy India 2027", 2027, "MISSING_EDITION"), + ("PyData Lagos 2027", 2027, "NEW_SERIES"), + ("“PyCon NP”, “PyCon Nepal” in Nepal", None, "NEW_SERIES"), + ("Brand New Conference", None, "NEW_SERIES"), + ] + + def test_record_shape(self, index): + record = desk_leads.listing_records(["SciPy India 2027"], LISTING_URL, index)[0] + assert record["source"] == "changedetection" + assert record["kind"] == "candidate" + assert record["url"] == LISTING_URL + assert record["payload"]["flags"] == ["listing"] + assert record["payload"]["names"] == ["SciPy India 2027"] + assert record["payload"]["fields"] == {"conference": "SciPy India", "year": 2027} + assert record["payload"]["matched"]["conference"] == "SciPy India" + assert normalise_record(record, NOW)["kind"] == "candidate" + + def test_cli_reads_lines_and_writes_records(self, tmp_path, capsys): + write_data(tmp_path) + lines = tmp_path / "lines.txt" + lines.write_text("\n".join(LISTING_LINES) + "\n", encoding="utf-8") + out = tmp_path / "records.jsonl" + args = ["listing", "--base", str(tmp_path), "--lines", str(lines), "--url", LISTING_URL, "--out", str(out)] + assert main(args) == 0 + assert len(out.read_text(encoding="utf-8").splitlines()) == 4 + assert "desk_leads: listing: 4 record(s)" in capsys.readouterr().err + + def test_records_reach_their_skills(self, tmp_path, index): + data = dataset() + records = desk_leads.listing_records(LISTING_LINES, LISTING_URL, index) + assert {result.get("status") for result in add_items(tmp_path, records, NOW)} == {"added"} + worklist = build_worklist(data, load_queue(tmp_path), TODAY, scout=False) + skills = {item["conference"]: item["skill"] for item in worklist["items"] if item["queue_refs"]} + assert skills["SciPy India"] == "new-year-checker" + assert skills["PyData Lagos"] == "scouter" + assert set(skills.values()) <= {"new-year-checker", "scouter"} + + def test_cli_listing_needs_lines_and_url(self, tmp_path): + with pytest.raises(SystemExit): + main(["listing", "--base", str(tmp_path), "--url", LISTING_URL]) + with pytest.raises(SystemExit): + main(["pretalx", "--base", str(tmp_path), "--lines", str(tmp_path / "x")]) + + class TestCli: def test_appends_json_lines_and_counts(self, tmp_path, web, sleeps, capsys): write_data(tmp_path) diff --git a/utils/desk_leads.py b/utils/desk_leads.py index 65421f63fb..ead964bc79 100644 --- a/utils/desk_leads.py +++ b/utils/desk_leads.py @@ -15,6 +15,10 @@ - ``registry``: PyCon names on the PSF trademark list, and events in the pycon.org registry, that the data does not track (kind ``registry``). ``psf-trademarks`` and ``pycon-org`` read one of the two each. +- ``listing``: the lines a changedetection watch on a page that lists many + events saw added, one event per line (kind ``candidate``, flag ``listing``), + for the editions and series the data does not track. ``--lines`` names the + file, ``--url`` the page. A find is tracked when it is the same series as an entry in any of the three data files, by the rules in AGENTS.md "Names": its name (year and a trailing @@ -1093,6 +1097,87 @@ def developers_events_records(events: Iterable[Any], index: TrackingIndex, today return records +def listing_names(line: str) -> list[str]: + """Return the event names on one line of a watched listing page. + + Parameters + ---------- + line : str + One line, e.g. "PyData Amsterdam" or ``"PyCon AT", "PyCon Austria" in Austria``. + + Returns + ------- + list[str] + The quoted names when the line quotes any, else the whole line. + """ + quoted = [name.strip() for name in re.findall(r"[\"“]([^\"“”]+)[\"”]", line) if name.strip()] + return quoted or [line.strip()] + + +def listing_records(lines: Iterable[str], page_url: str, index: TrackingIndex) -> list[dict]: + """Turn the lines a watched listing page added into candidate records. + + A changedetection watch on a page that lists many events keeps one event + per line, so each added line is one find. A line with a year is checked + edition by edition; a line without one (a series registry, or a list + that names no years) only says whether the series exists. + + Parameters + ---------- + lines : Iterable[str] + The added lines. + page_url : str + The watched page, which is where each find came from. + index : TrackingIndex + The data's series. + + Returns + ------- + list[dict] + One record per line whose series, or whose edition for the line's + year, the data does not track, in line order. + """ + records = [] + seen: set[str] = set() + for raw in lines: + # diff_added_clean has no markers; strip them anyway in case a body sends diff_added + line = re.sub(r"^(?:\(added\)|\+)\s*", "", " ".join(raw.split())) + if not line or line in seen: + continue + seen.add(line) + names = listing_names(line) + match = NAME_YEAR.search(line) + year = int(match[0]) if match else None + if year is None: + if any(index.is_tracked_name(name) for name in names): + continue + found = {"classification": "NEW_SERIES", "matched": None} + else: + results = [index.classify(name, year, None) for name in names] + if any(result is None for result in results): + continue + found = next((r for r in results if r["classification"] != "NEW_SERIES"), results[0]) + records.append( + { + "source": "changedetection", + "kind": "candidate", + "title": line[:200], + "url": page_url, + "year": year, + "payload": { + "classification": found["classification"], + "fields": {"conference": clean_name(names[0]), "year": year}, + "source_id": f"{page_url}#{line}", + "evidence_url": page_url, + "matched": found["matched"], + "flags": ["listing"], + "names": names, + }, + }, + ) + return records + + def developers_events_leads(index: TrackingIndex, today: date) -> list[dict]: """Fetch developers.events and return its candidate records. @@ -1419,7 +1504,8 @@ def main(argv: list[str] | None = None) -> int: before the failure are still written. """ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("source", choices=SOURCES) + # listing reads the lines a watch saw added, not a source of its own, so collect() leaves it out + parser.add_argument("source", choices=(*SOURCES, "listing")) parser.add_argument("--base", type=Path, default=Path(), help="Repository root (default: current directory)") parser.add_argument( "--today", @@ -1429,14 +1515,32 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--out", type=Path, help="File to append the records to (default: stdout)") parser.add_argument( "--url", - help="pretalx only: check just the entries on this CfP page, which may carry a watch's year template", + help=( + "pretalx: check just the entries on this CfP page, which may carry a watch's year template; " + "listing: the watched page the lines came from" + ), ) + parser.add_argument("--lines", type=Path, help="listing only: file with the lines the watched page added") args = parser.parse_args(argv) + if args.source == "listing": + if args.lines is None or args.url is None: + parser.error("the listing source needs --lines and --url") + try: + lines = args.lines.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as error: + parser.error(f"cannot read {args.lines}: {error}") + records = listing_records(lines, args.url, TrackingIndex.from_data(load_data(args.base))) + write_records(records, args.out) + print(f"desk_leads: listing: {len(records)} record(s)", file=sys.stderr) + return 0 + if args.lines is not None: + parser.error("--lines works with the listing source only") + slug = None if args.url is not None: if args.source != "pretalx": - parser.error("--url works with the pretalx source only") + parser.error("--url works with the pretalx and listing sources only") slug = pretalx_slug(resolve_watch_url(args.url)) if slug is None: print(f"desk_leads: pretalx: {args.url} is not a pretalx.com event page, nothing to check", file=sys.stderr) diff --git a/utils/desk_queue.py b/utils/desk_queue.py index d8077ac51c..ed79600ac4 100644 --- a/utils/desk_queue.py +++ b/utils/desk_queue.py @@ -139,8 +139,8 @@ | `pretalx` | `drift` | `conference`, `year`, `diffs` | | `pretalx`, `developers-events`, `canonical-import` | `candidate` | `classification`, `fields`, `source_id`, \ `evidence_url`, `matched`, `flags` | +| `changedetection` | `candidate` | as above, with `names` and the flag `listing` | | `registry` | `registry` | `names`, `change`, `registry`, `evidence_url` | -| `changedetection` | `registry` | `registry` (`listing`), `diff`, `watch_uuid`, `trigger` | A `drift` diff maps a field to `{"data", "source"}`: the value in the data and what the source states. A candidate's `classification` is @@ -149,9 +149,10 @@ `flags` say), and `matched` names the entry it matched, or is `null`. The worklist routes a candidate by its own match against the data, not by its classification. A `lead-only` flag means the source's licence keeps it to -leads: use only its name, year and link. `registry` is `psf-trademarks` or -`pycon.org`, and a pycon.org item adds `website`; from changedetection it is -`listing`, a watched page that lists many events, whose `diff` holds the change. +leads: use only its name, year and link. A `listing` flag marks one line a +changedetection watch on a page that lists many events added, with `names` +the names on that line. `registry` is `psf-trademarks` or `pycon.org`; a +pycon.org item adds `website`. Every item is data for the desk to check on the live sources. Nothing in a queue item reaches the conference data without that check. From fc15f35c7c07087d85bb6aac707d26d74272451e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:37:11 +0100 Subject: [PATCH 2/3] Read the PSF trademark list through its listing watch, not a parser of its own The changedetection watch on the trademark page now keeps one list item per line, so its diff goes through the listing check like the other listing pages. That retires the psf-trademarks source: the page API call, trademark_items and psf_records. With pycon.org the only registry left, the registry source that read both goes too, and desk-leads.yml runs pycon-org as its own step. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGjzRsX9nEb4swsKYms --- .github/workflows/desk-leads.yml | 66 ++++++--------- tests/test_desk_leads.py | 112 ++++++-------------------- utils/desk_leads.py | 133 +++++-------------------------- utils/desk_queue.py | 5 +- 4 files changed, 73 insertions(+), 243 deletions(-) diff --git a/.github/workflows/desk-leads.yml b/.github/workflows/desk-leads.yml index c7782a1728..67811dedb8 100644 --- a/.github/workflows/desk-leads.yml +++ b/.github/workflows/desk-leads.yml @@ -1,7 +1,7 @@ name: Desk Leads -# Report-only. Looks for leads on pretalx, developers.events and the PyCon -# registries and queues them for the desk, which checks each one on the +# Report-only. Looks for leads on pretalx, developers.events and the pycon.org +# registry and queues them for the desk, which checks each one on the # conference's own pages. Nothing here writes the conference data. # # Sources run when changedetection.io sees them change, with a slow schedule @@ -12,11 +12,13 @@ name: Desk Leads # robots.txt on pretalx.com disallows /api/. A watch on one CfP page sends # pretalx-change with that page's URL, and only its entries are checked. # Weekly, all of them and the next-edition guess. -# - developers.events (CC BY-NC data, used for leads only), the PSF PyCon -# trademark list and the pycon.org registry: a watch on each (the Python -# rows of TAGS.csv, the trademark page, the registry's latest commit) sends -# registry-change naming its source, and that source is read in full. -# Monthly, all three. +# - developers.events (CC BY-NC data, used for leads only) and the pycon.org +# registry: a watch on each (the Python rows of TAGS.csv, the registry's +# latest commit) sends registry-change naming its source, and that source +# is read in full. Monthly, both. +# +# The PSF PyCon trademark list is a listing page: its watch sends +# listing-change to check-conference-update.yml, which checks each added line. # # A finding the desk has handled is not queued again until it changes, so a # lead that comes back every run adds nothing. @@ -24,7 +26,7 @@ name: Desk Leads on: schedule: - cron: '23 4 * * 1' # Mondays 04:23 UTC: pretalx - - cron: '41 4 1 * *' # the 1st of the month, 04:41 UTC: developers-events and both registries + - cron: '41 4 1 * *' # the 1st of the month, 04:41 UTC: developers-events and pycon-org repository_dispatch: types: [registry-change, pretalx-change] workflow_dispatch: @@ -36,8 +38,6 @@ on: - all - pretalx - developers-events - - registry - - psf-trademarks - pycon-org default: all @@ -76,7 +76,7 @@ jobs: if [ "$EVENT_NAME" = "schedule" ]; then case "$SCHEDULE" in '23 4 * * 1') SELECTED="pretalx" ;; - '41 4 1 * *') SELECTED="developers-events psf-trademarks pycon-org" ;; + '41 4 1 * *') SELECTED="developers-events pycon-org" ;; *) echo "::error title=Desk leads::Unknown schedule $SCHEDULE"; exit 1 ;; esac elif [ "$EVENT_NAME" = "repository_dispatch" ] && [ "$ACTION" = "pretalx-change" ]; then @@ -89,32 +89,22 @@ jobs: esac elif [ "$EVENT_NAME" = "repository_dispatch" ]; then case "$PAYLOAD_SOURCE" in - developers-events|psf-trademarks|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;; + developers-events|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;; *) echo "::error title=Desk leads::registry-change names no known source"; exit 1 ;; esac elif [ "${CHOSEN:-all}" = "all" ]; then - SELECTED="pretalx developers-events psf-trademarks pycon-org" - elif [ "$CHOSEN" = "registry" ]; then - SELECTED="psf-trademarks pycon-org" + SELECTED="pretalx developers-events pycon-org" else SELECTED="$CHOSEN" fi echo "Sources: $SELECTED" - for source in pretalx developers-events; do + for source in pretalx developers-events pycon-org; do case " $SELECTED " in *" $source "*) RUN=true ;; *) RUN=false ;; esac echo "${source//-/_}=$RUN" >> "$GITHUB_OUTPUT" done - # The registries run in one step, each on its own - REGISTRIES="" - for registry in psf-trademarks pycon-org; do - case " $SELECTED " in - *" $registry "*) REGISTRIES="$REGISTRIES $registry" ;; - esac - done - echo "registries=${REGISTRIES# }" >> "$GITHUB_OUTPUT" echo "page=$PAGE" >> "$GITHUB_OUTPUT" # A source that fails gets a warning; the other sources still run. @@ -156,22 +146,18 @@ jobs: echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT" echo "status=$STATUS" >> "$GITHUB_OUTPUT" - - name: PyCon registries - id: registry - if: steps.plan.outputs.registries != '' - env: - REGISTRIES: ${{ steps.plan.outputs.registries }} + - name: pycon.org + id: pycon_org + if: steps.plan.outputs.pycon_org == 'true' run: | set -uo pipefail touch "$RUNNER_TEMP/desk-records.jsonl" BEFORE=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl") STATUS=ok - for registry in $REGISTRIES; do - if ! pixi run python utils/desk_leads.py "$registry" --out "$RUNNER_TEMP/desk-records.jsonl"; then - STATUS=failed - echo "::warning title=Desk leads::$registry failed; what it found is still queued" - fi - done + if ! pixi run python utils/desk_leads.py pycon-org --out "$RUNNER_TEMP/desk-records.jsonl"; then + STATUS=failed + echo "::warning title=Desk leads::pycon-org failed; what it found is still queued" + fi AFTER=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl") echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT" echo "status=$STATUS" >> "$GITHUB_OUTPUT" @@ -183,7 +169,7 @@ jobs: uses: ./.github/actions/desk-queue-write with: records: ${{ runner.temp }}/desk-records.jsonl - message: Queue leads from pretalx, developers.events and the PyCon registries + message: Queue leads from pretalx, developers.events and pycon.org - name: Summary if: ${{ !cancelled() }} @@ -192,8 +178,8 @@ jobs: PRETALX_STATUS: ${{ steps.pretalx.outputs.status }} DEVELOPERS_EVENTS_COUNT: ${{ steps.developers_events.outputs.count }} DEVELOPERS_EVENTS_STATUS: ${{ steps.developers_events.outputs.status }} - REGISTRY_COUNT: ${{ steps.registry.outputs.count }} - REGISTRY_STATUS: ${{ steps.registry.outputs.status }} + PYCON_ORG_COUNT: ${{ steps.pycon_org.outputs.count }} + PYCON_ORG_STATUS: ${{ steps.pycon_org.outputs.status }} ADDED: ${{ steps.queue.outputs.added }} run: | row() { @@ -210,7 +196,7 @@ jobs: echo "| --- | --- | --- |" row pretalx "$PRETALX_COUNT" "$PRETALX_STATUS" row developers-events "$DEVELOPERS_EVENTS_COUNT" "$DEVELOPERS_EVENTS_STATUS" - row registry "$REGISTRY_COUNT" "$REGISTRY_STATUS" + row pycon-org "$PYCON_ORG_COUNT" "$PYCON_ORG_STATUS" echo "" echo "- New in the desk queue: ${ADDED:-0}" } >> "$GITHUB_STEP_SUMMARY" @@ -219,7 +205,7 @@ jobs: - name: Fail when a source failed if: ${{ !cancelled() }} env: - STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.registry.outputs.status }} + STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.pycon_org.outputs.status }} run: | case " $STATUSES " in *" failed "*) echo "::error title=Desk leads::a source failed, see the warnings above"; exit 1 ;; diff --git a/tests/test_desk_leads.py b/tests/test_desk_leads.py index a26d435ceb..f23ddc40ac 100644 --- a/tests/test_desk_leads.py +++ b/tests/test_desk_leads.py @@ -1,4 +1,4 @@ -"""Tests for desk_leads: lead importers for pretalx, developers.events and the PyCon registries. +"""Tests for desk_leads: lead importers for pretalx, developers.events, pycon.org and listing pages. Nothing here touches the network: every request goes to the responses mock, and opening a socket fails the test. @@ -38,7 +38,6 @@ from desk_leads import pretalx_slug from desk_leads import pycon_org_record from desk_leads import registry_paths -from desk_leads import trademark_items from desk_queue import add_items from desk_queue import normalise_record from desk_worklist import build_worklist @@ -731,22 +730,6 @@ def test_whole_source_failure_exits_1(self, tmp_path, web, capsys): assert "503" in capsys.readouterr().err -PSF_CONTENT = """\ -

PyCon Conferences around the World

-

The following are national-level and regional authorized PyCon conferences:

-
-
    -
  • "Kiwi PyCon", "PyCon NZ" in New Zealand
    • -
  • "PyCon MEA (Middle East & Africa)" in Dubai
    • -
  • "PyCon NP", "PyCon Nepal" in Nepal
    • -
  • "PyCon SK", "PyCon Slovakia" in Slovakia
    • -
  • "PyCon VE", "PyCon Venezuela" in Venezuela
    • -
-
-

If we have forgotten a national or regional PyCon conference in this list, please write to us.

-
  • "Not a PyCon" in a later list
-""" - REGISTRY_FILES = { "/_national/pycon-ireland.yml": "---\nname: PyCon Ireland\nflag: ie\nlocation: Ireland\nwebsite: https://python.ie\n---\n", "/_national/pycon-dk.yml": ( @@ -764,11 +747,10 @@ def test_whole_source_failure_exits_1(self, tmp_path, web, capsys): } -def registry_web(web, psf_status=200, broken=None): - if psf_status == 200: - web.add(responses.GET, desk_leads.PSF_TRADEMARKS_API, json={"content": PSF_CONTENT, "path": "/psf/pycon/"}) - else: - web.add(responses.GET, desk_leads.PSF_TRADEMARKS_API, status=psf_status) +def registry_web(web, listing_status=200, broken=None): + if listing_status != 200: + web.add(responses.GET, desk_leads.PYCON_ORG_FILES, status=listing_status) + return listing = [{"name": path, "hash": "x", "size": 90} for path in REGISTRY_FILES] listing += [{"name": "/index.html", "hash": "x", "size": 9}, {"name": "/_national/README.md", "hash": "x"}] web.add(responses.GET, desk_leads.PYCON_ORG_FILES, json={"type": "gh", "files": listing}) @@ -780,20 +762,7 @@ def registry_web(web, psf_status=200, broken=None): web.add(responses.GET, url, body=text) -class TestRegistry: - def test_trademark_items(self): - assert trademark_items(PSF_CONTENT) == [ - ["Kiwi PyCon", "PyCon NZ"], - ["PyCon MEA (Middle East & Africa)"], - ["PyCon NP", "PyCon Nepal"], - ["PyCon SK", "PyCon Slovakia"], - ["PyCon VE", "PyCon Venezuela"], - ] - - def test_trademark_page_without_the_list(self): - with pytest.raises(ValueError, match="no list"): - trademark_items("

Nothing here

") - +class TestPyconOrg: def test_front_matter(self): assert front_matter(REGISTRY_FILES["/_regional/pygotham.yml"]) == { "name": "PyGotham", @@ -832,53 +801,20 @@ def test_registry_paths(self): with pytest.raises(ValueError, match="no files"): registry_paths({"files": None}) - def test_both_registries(self, index, web): + def test_untracked_events(self, index, web): registry_web(web) - records = desk_leads.registry_leads(index) - assert [(record["payload"]["registry"], record["payload"]["names"]) for record in records] == [ - ("psf-trademarks", ["PyCon NP", "PyCon Nepal"]), - ("psf-trademarks", ["PyCon VE", "PyCon Venezuela"]), - ("pycon.org", ["PyGotham"]), - ] - assert records[0] == { - "source": "registry", - "kind": "registry", - "title": "PyCon NP", - "url": "https://www.python.org/psf/trademarks/pycon/", - "year": None, - "payload": { - "names": ["PyCon NP", "PyCon Nepal"], - "change": "untracked", - "registry": "psf-trademarks", - "evidence_url": "https://www.python.org/psf/trademarks/pycon/", - }, - } + records = desk_leads.collect("pycon-org", dataset(), index, TODAY) + assert [record["title"] for record in records] == ["PyGotham"] def test_one_failed_file_is_skipped(self, index, web, capsys): registry_web(web, broken="/_regional/pygotham.yml") - records = desk_leads.registry_leads(index) - assert [record["title"] for record in records] == ["PyCon NP", "PyCon VE"] + assert desk_leads.pycon_org_leads(index) == [] assert "pygotham.yml" in capsys.readouterr().err - def test_failed_registry_keeps_the_other(self, index, web): - registry_web(web, psf_status=500) - with pytest.raises(SourceError, match="psf-trademarks") as raised: - desk_leads.registry_leads(index) - assert [record["title"] for record in raised.value.records] == ["PyGotham"] - - @pytest.mark.parametrize( - ("source", "titles"), - [("psf-trademarks", ["PyCon NP", "PyCon VE"]), ("pycon-org", ["PyGotham"])], - ) - def test_each_registry_on_its_own(self, index, web, source, titles): - registry_web(web) - records = desk_leads.collect(source, dataset(), index, TODAY) - assert [record["title"] for record in records] == titles - - def test_one_registry_failing_alone_is_an_error(self, index, web): - registry_web(web, psf_status=500) - with pytest.raises(SourceError, match="psf-trademarks") as raised: - desk_leads.collect("psf-trademarks", dataset(), index, TODAY) + def test_failed_listing_is_an_error(self, index, web): + registry_web(web, listing_status=500) + with pytest.raises(SourceError, match="pycon.org") as raised: + desk_leads.collect("pycon-org", dataset(), index, TODAY) assert raised.value.records == [] @@ -1026,14 +962,19 @@ def test_url_off_pretalx_checks_nothing(self, tmp_path, web, capsys): def test_url_needs_the_pretalx_source(self, tmp_path): with pytest.raises(SystemExit): - main(["registry", "--base", str(tmp_path), "--url", "https://pretalx.com/x/cfp"]) + main(["pycon-org", "--base", str(tmp_path), "--url", "https://pretalx.com/x/cfp"]) + + @pytest.mark.parametrize("source", ["registry", "psf-trademarks"]) + def test_retired_sources_are_refused(self, tmp_path, source): + with pytest.raises(SystemExit): + main([source, "--base", str(tmp_path)]) - def test_failed_registry_still_writes_the_other(self, tmp_path, web, capsys): + def test_failed_source_exits_1(self, tmp_path, web, capsys): write_data(tmp_path) - registry_web(web, psf_status=500) + registry_web(web, listing_status=500) out = tmp_path / "records.jsonl" - assert main(["registry", "--base", str(tmp_path), "--out", str(out)]) == 1 - assert [json.loads(line)["title"] for line in out.read_text(encoding="utf-8").splitlines()] == ["PyGotham"] + assert main(["pycon-org", "--base", str(tmp_path), "--out", str(out)]) == 1 + assert "pycon.org" in capsys.readouterr().err class TestQueueRoundTrip: @@ -1046,8 +987,7 @@ def test_records_route_to_their_skills(self, tmp_path, web, sleeps): data = dataset() index = TrackingIndex.from_data(data) records = [] - # "registry" reads both registries, so their own sources would only repeat it - for source in ("pretalx", "developers-events", "registry"): + for source in ("pretalx", "developers-events", "pycon-org"): records.extend(desk_leads.collect(source, data, index, TODAY)) for record in records: @@ -1072,8 +1012,6 @@ def test_records_route_to_their_skills(self, tmp_path, web, sleeps): (("candidate",), "PyCon Greece", 2027): "new-year-checker", (("candidate",), "PyData Lagos", 2027): "scouter", (("candidate",), "Python Hellas Summit", 2027): "scouter", - (("registry",), "PyCon Nepal", None): "scouter", - (("registry",), "PyCon Venezuela", None): "scouter", (("registry",), "PyGotham", None): "scouter", } diff --git a/utils/desk_leads.py b/utils/desk_leads.py index ead964bc79..0c1540adfb 100644 --- a/utils/desk_leads.py +++ b/utils/desk_leads.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Find leads for the deadline desk on pretalx, developers.events and the PyCon registries. +"""Find leads for the deadline desk on pretalx, developers.events and the pycon.org registry. Each source writes queue records as JSON lines, which utils/desk_queue.py adds to the desk-queue branch. Nothing here writes the data files: the desk checks @@ -12,13 +12,12 @@ robots.txt of pretalx.com disallows ``/api/``. - ``developers-events``: upcoming Python events on developers.events that the data does not track (kind ``candidate``). -- ``registry``: PyCon names on the PSF trademark list, and events in the - pycon.org registry, that the data does not track (kind ``registry``). - ``psf-trademarks`` and ``pycon-org`` read one of the two each. +- ``pycon-org``: events in the pycon.org registry that the data does not + track (kind ``registry``). - ``listing``: the lines a changedetection watch on a page that lists many events saw added, one event per line (kind ``candidate``, flag ``listing``), for the editions and series the data does not track. ``--lines`` names the - file, ``--url`` the page. + file, ``--url`` the page. The PSF PyCon trademark list comes this way. A find is tracked when it is the same series as an entry in any of the three data files, by the rules in AGENTS.md "Names": its name (year and a trailing @@ -37,11 +36,10 @@ python utils/desk_leads.py pretalx --out records.jsonl python utils/desk_leads.py developers-events --today 2026-10-14 - python utils/desk_leads.py registry --base . --out records.jsonl + python utils/desk_leads.py pycon-org --base . --out records.jsonl """ import argparse -import html import json import re import sys @@ -75,9 +73,7 @@ from .sort_yaml import DATEFORMAT from .tidy_conf.yaml import load_title_mappings -SOURCES = ("pretalx", "developers-events", "registry", "psf-trademarks", "pycon-org") -# The two registries "registry" reads together; each also runs on its own -REGISTRIES = ("psf-trademarks", "pycon-org") +SOURCES = ("pretalx", "developers-events", "pycon-org") DATA_FILES = ("conferences", "archive", "legacy") USER_AGENT = "pythondeadlin.es desk leads (+https://pythondeadlin.es)" TIMEOUT = 20 @@ -97,9 +93,6 @@ PRETALX_CFP = "https://pretalx.com/{slug}/cfp" DEVELOPERS_EVENTS = "https://developers.events/all-events.json" -PSF_TRADEMARKS = "https://www.python.org/psf/trademarks/pycon/" -PSF_TRADEMARKS_API = "https://www.python.org/api/v1/pages/page/852/?format=json" -PSF_LIST_MARKER = "national-level and regional authorized PyCon conferences" PYCON_ORG_FILES = "https://data.jsdelivr.com/v1/packages/gh/PyCon/pycon.org@master?structure=flat" PYCON_ORG_RAW = "https://raw.githubusercontent.com/PyCon/pycon.org/master" @@ -1204,73 +1197,6 @@ def developers_events_leads(index: TrackingIndex, today: date) -> list[dict]: return developers_events_records(events, index, today) -def trademark_items(content: str) -> list[list[str]]: - """Read the authorised PyCon names from the PSF trademark page. - - Parameters - ---------- - content : str - The page HTML. - - Returns - ------- - list[list[str]] - The quoted names of each list item after "national-level and regional - authorized PyCon conferences", in page order. - - Raises - ------ - ValueError - When the page has no such list. - """ - start = content.find(PSF_LIST_MARKER) - list_start = content.find("= 0 else -1 - list_end = content.find("", list_start) if list_start >= 0 else -1 - if list_end < 0: - raise ValueError("no list of authorised PyCon conferences") - items = [] - for raw in re.findall(r"]*>(.*?)", content[list_start:list_end], re.DOTALL): - text = re.sub(r"<[^>]+>", "", html.unescape(raw)) - names = [" ".join(name.split()) for name in re.findall(r'"([^"]+)"', text)] - if names: - items.append(names) - return items - - -def psf_records(items: Iterable[list[str]], index: TrackingIndex) -> list[dict]: - """Return a registry record for each trademark list item the data does not track. - - Parameters - ---------- - items : Iterable[list[str]] - Names per list item, from trademark_items. - index : TrackingIndex - The data's series. - - Returns - ------- - list[dict] - Records for items none of whose names is tracked. - """ - return [ - { - "source": "registry", - "kind": "registry", - "title": names[0], - "url": PSF_TRADEMARKS, - "year": None, - "payload": { - "names": list(names), - "change": "untracked", - "registry": "psf-trademarks", - "evidence_url": PSF_TRADEMARKS, - }, - } - for names in items - if names and not any(index.is_tracked_name(name) for name in names) - ] - - def front_matter(text: str) -> dict | None: """Parse the YAML front matter of a pycon.org registry file. @@ -1361,63 +1287,44 @@ def pycon_org_record(meta: dict, raw_url: str, index: TrackingIndex) -> dict | N } -def registry_leads(index: TrackingIndex, parts: Iterable[str] = REGISTRIES) -> list[dict]: - """Read the PSF trademark list, the pycon.org registry, or both. +def pycon_org_leads(index: TrackingIndex) -> list[dict]: + """Read the pycon.org registry. Parameters ---------- index : TrackingIndex The data's series. - parts : Iterable[str], optional - Which registries to read, of REGISTRIES; by default both. Returns ------- list[dict] - Registry records: the trademark list's, then pycon.org's. + Registry records for the events the data does not track. Raises ------ SourceError - When a registry cannot be read, carrying the other's records. + When the registry's file listing cannot be read. """ - parts = set(parts) - records: list[dict] = [] - errors = [] - if "psf-trademarks" in parts: - try: - page = fetch_json(PSF_TRADEMARKS_API) - content = page.get("content") if isinstance(page, dict) else None - if not isinstance(content, str): - raise ValueError("no content in the page JSON") - records.extend(psf_records(trademark_items(content), index)) - except (SourceError, ValueError) as error: - errors.append(f"psf-trademarks: {error}") - - paths = [] - if "pycon-org" in parts: - try: - paths = registry_paths(fetch_json(PYCON_ORG_FILES)) - except (SourceError, ValueError) as error: - errors.append(f"pycon.org: {error}") + try: + paths = registry_paths(fetch_json(PYCON_ORG_FILES)) + except (SourceError, ValueError) as error: + raise SourceError(f"pycon.org: {error}") from error + records = [] for path in paths: raw_url = f"{PYCON_ORG_RAW}{path}" try: response = fetch(raw_url) response.raise_for_status() except requests.RequestException as error: - report(f"registry: {raw_url}: {first_line(error)}, skipped") + report(f"pycon-org: {raw_url}: {first_line(error)}, skipped") continue meta = front_matter(response.text) if meta is None: - report(f"registry: {raw_url} has no front matter, skipped") + report(f"pycon-org: {raw_url} has no front matter, skipped") continue record = pycon_org_record(meta, raw_url, index) if record is not None: records.append(record) - - if errors: - raise SourceError("; ".join(errors), records) return records @@ -1449,10 +1356,8 @@ def collect(source: str, data: dict[str, list[dict]], index: TrackingIndex, toda records = pretalx_leads(data, index, today) elif source == "developers-events": records = developers_events_leads(index, today) - elif source == "registry": - records = registry_leads(index) - elif source in REGISTRIES: - records = registry_leads(index, (source,)) + elif source == "pycon-org": + records = pycon_org_leads(index) else: raise ValueError(f"unknown source {source!r}") return _unique(records) diff --git a/utils/desk_queue.py b/utils/desk_queue.py index ed79600ac4..467d5d5cda 100644 --- a/utils/desk_queue.py +++ b/utils/desk_queue.py @@ -151,8 +151,9 @@ classification. A `lead-only` flag means the source's licence keeps it to leads: use only its name, year and link. A `listing` flag marks one line a changedetection watch on a page that lists many events added, with `names` -the names on that line. `registry` is `psf-trademarks` or `pycon.org`; a -pycon.org item adds `website`. +the names on that line. `registry` is `pycon.org`, and the item adds +`website`; older items can say `psf-trademarks`, from before the trademark +list became a listing page. Every item is data for the desk to check on the live sources. Nothing in a queue item reaches the conference data without that check. From bb2b11e87b1c5f33c310bff47588d822830700c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:39:43 +0100 Subject: [PATCH 3/3] Name the pycon.org source once in its error, and test the error without a hostname CodeQL read the test's substring check on pycon.org as URL sanitisation. The tests now check the HTTP error and the summary line instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGjzRsX9nEb4swsKYms --- tests/test_desk_leads.py | 4 ++-- utils/desk_leads.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_desk_leads.py b/tests/test_desk_leads.py index f23ddc40ac..1e99fc46a5 100644 --- a/tests/test_desk_leads.py +++ b/tests/test_desk_leads.py @@ -813,7 +813,7 @@ def test_one_failed_file_is_skipped(self, index, web, capsys): def test_failed_listing_is_an_error(self, index, web): registry_web(web, listing_status=500) - with pytest.raises(SourceError, match="pycon.org") as raised: + with pytest.raises(SourceError, match="500 Server Error") as raised: desk_leads.collect("pycon-org", dataset(), index, TODAY) assert raised.value.records == [] @@ -974,7 +974,7 @@ def test_failed_source_exits_1(self, tmp_path, web, capsys): registry_web(web, listing_status=500) out = tmp_path / "records.jsonl" assert main(["pycon-org", "--base", str(tmp_path), "--out", str(out)]) == 1 - assert "pycon.org" in capsys.readouterr().err + assert "pycon-org: 0 record(s)" in capsys.readouterr().err class TestQueueRoundTrip: diff --git a/utils/desk_leads.py b/utils/desk_leads.py index 0c1540adfb..20e1d9e4e2 100644 --- a/utils/desk_leads.py +++ b/utils/desk_leads.py @@ -1305,10 +1305,11 @@ def pycon_org_leads(index: TrackingIndex) -> list[dict]: SourceError When the registry's file listing cannot be read. """ + listing = fetch_json(PYCON_ORG_FILES) try: - paths = registry_paths(fetch_json(PYCON_ORG_FILES)) - except (SourceError, ValueError) as error: - raise SourceError(f"pycon.org: {error}") from error + paths = registry_paths(listing) + except ValueError as error: + raise SourceError(f"{PYCON_ORG_FILES}: {error}") from error records = [] for path in paths: raw_url = f"{PYCON_ORG_RAW}{path}"