diff --git a/.claude/skills/scouter/SKILL.md b/.claude/skills/scouter/SKILL.md index 0329574529..fd76bfa9dc 100644 --- a/.claude/skills/scouter/SKILL.md +++ b/.claude/skills/scouter/SKILL.md @@ -14,7 +14,7 @@ The desk hands you the daily `scout` item, or an item about a series no data fil - `lead-only`: an aggregator lead. Copy no value from it; take every value from a primary source. - `no-website`: the shared sheet has this edition but no website. Search for one. - `denylist-review: : `: the domain is on the denylist's `review` list. Copy the flag into the `flags` of the find's change set; the critic returns needs-maintainer. - - A `registry` item whose payload `registry` is `listing`: a watched page that lists many events changed. Each event name or link the payload's `diff` adds is a find, with the item's `url` as where it came from; fetch that page when the diff is cut short or names an event without a link. A listing page is a lead, never the evidence (step 4). + - `listing`: one line a watched page that lists many events added, such as "PyData Amsterdam". The item's `url` is that page and its payload `names` the names on the line. Find the event on the page or by search. A listing page is a lead, never the evidence (step 4). - A `maintainer-request` item: the series and URLs the comment names are the find, and the request settles its scope (step 2). - The `scout` item: search each of these, with `` standing for this year and next (the year of the worklist's `today`, which the desk gives with the item). - Targeted web searches: "PyCon ", "PyData ", "Python conference call for proposals", "PyDay", "Python Pizza", framework and scientific library names with "conference", and the same in other languages ("conferencia Python", "Python Konferenz"). diff --git a/.github/workflows/check-conference-update.yml b/.github/workflows/check-conference-update.yml index bdf417ef63..8f57fc8707 100644 --- a/.github/workflows/check-conference-update.yml +++ b/.github/workflows/check-conference-update.yml @@ -6,9 +6,11 @@ name: Check Conference Update # # conference-change comes from a watch on one conference's pages and queues a # signal for that series. listing-change comes from a watch on a page that -# lists many events (pydata.org/upcoming-events, barcamps.eu and the like) and -# queues a registry item, which the worklist always gives the scouter: the -# names and links the diff adds are the finds. +# lists many events (pydata.org/upcoming-events, conference.scipy.org, the +# PSF PyCon trademark list), filtered to one event per line. Each added line +# is checked against the data in code (utils/desk_leads.py listing): a tracked +# edition is dropped, and the rest are queued as candidates, which the +# worklist routes to the new-year-checker or the scouter. on: # Triggered by changedetection.io webhook @@ -101,16 +103,26 @@ jobs: name, year = clean_watch_title(title) name = " ".join(name.split()) # one line, for the step output - payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger} if listing: - payload["registry"] = "listing" + # One event per added line: the next step checks each against + # the data. Files, not step outputs, carry the page's text. + temp = os.environ["RUNNER_TEMP"] + with open(f"{temp}/listing-lines.txt", "w", encoding="utf-8") as f: + f.write(diff if diff.endswith("\n") or not diff else diff + "\n") + with open(f"{temp}/listing-url.txt", "w", encoding="utf-8") as f: + f.write(" ".join(resolve_watch_url(url.strip()).split())) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"series={name}\nlisting=true\n") + print(f"Checking {len(diff.splitlines())} added line(s) from {name or title}") + sys.exit(0) + payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger} if event_name == "workflow_dispatch": # The run id gives each manual request its own queue id, so # asking again with the same inputs is not taken as handled. payload["requested"] = os.environ["GITHUB_RUN_ID"] record = { "source": "changedetection", - "kind": "registry" if listing else "signal", + "kind": "signal", "title": title.strip(), # A watch that follows a series sends its URL with the year template in it "url": resolve_watch_url(url.strip()), @@ -121,9 +133,24 @@ jobs: f.write(json.dumps(record, ensure_ascii=False) + "\n") with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: out.write(f"series={name}\n") - print(f"Queueing a {trigger} {record['kind']} item for {name or title}") + print(f"Queueing a {trigger} signal for {name or title}") BUILD + - name: Setup Pixi + if: steps.record.outputs.listing == 'true' + uses: prefix-dev/setup-pixi@v0.11.2 + + # Each added line is one event. A line whose edition (or, without a + # year, whose series) the data tracks is dropped; the rest become + # candidates that the worklist routes like any other. + - name: Check the listing lines against the data + if: steps.record.outputs.listing == 'true' + run: | + pixi run python utils/desk_leads.py listing \ + --lines "$RUNNER_TEMP/listing-lines.txt" \ + --url "$(cat "$RUNNER_TEMP/listing-url.txt")" \ + --out "$RUNNER_TEMP/desk-records.jsonl" + - name: Write to the desk queue id: queue uses: ./.github/actions/desk-queue-write diff --git a/.github/workflows/desk-leads.yml b/.github/workflows/desk-leads.yml index c7782a1728..67811dedb8 100644 --- a/.github/workflows/desk-leads.yml +++ b/.github/workflows/desk-leads.yml @@ -1,7 +1,7 @@ name: Desk Leads -# Report-only. Looks for leads on pretalx, developers.events and the PyCon -# registries and queues them for the desk, which checks each one on the +# Report-only. Looks for leads on pretalx, developers.events and the pycon.org +# registry and queues them for the desk, which checks each one on the # conference's own pages. Nothing here writes the conference data. # # Sources run when changedetection.io sees them change, with a slow schedule @@ -12,11 +12,13 @@ name: Desk Leads # robots.txt on pretalx.com disallows /api/. A watch on one CfP page sends # pretalx-change with that page's URL, and only its entries are checked. # Weekly, all of them and the next-edition guess. -# - developers.events (CC BY-NC data, used for leads only), the PSF PyCon -# trademark list and the pycon.org registry: a watch on each (the Python -# rows of TAGS.csv, the trademark page, the registry's latest commit) sends -# registry-change naming its source, and that source is read in full. -# Monthly, all three. +# - developers.events (CC BY-NC data, used for leads only) and the pycon.org +# registry: a watch on each (the Python rows of TAGS.csv, the registry's +# latest commit) sends registry-change naming its source, and that source +# is read in full. Monthly, both. +# +# The PSF PyCon trademark list is a listing page: its watch sends +# listing-change to check-conference-update.yml, which checks each added line. # # A finding the desk has handled is not queued again until it changes, so a # lead that comes back every run adds nothing. @@ -24,7 +26,7 @@ name: Desk Leads on: schedule: - cron: '23 4 * * 1' # Mondays 04:23 UTC: pretalx - - cron: '41 4 1 * *' # the 1st of the month, 04:41 UTC: developers-events and both registries + - cron: '41 4 1 * *' # the 1st of the month, 04:41 UTC: developers-events and pycon-org repository_dispatch: types: [registry-change, pretalx-change] workflow_dispatch: @@ -36,8 +38,6 @@ on: - all - pretalx - developers-events - - registry - - psf-trademarks - pycon-org default: all @@ -76,7 +76,7 @@ jobs: if [ "$EVENT_NAME" = "schedule" ]; then case "$SCHEDULE" in '23 4 * * 1') SELECTED="pretalx" ;; - '41 4 1 * *') SELECTED="developers-events psf-trademarks pycon-org" ;; + '41 4 1 * *') SELECTED="developers-events pycon-org" ;; *) echo "::error title=Desk leads::Unknown schedule $SCHEDULE"; exit 1 ;; esac elif [ "$EVENT_NAME" = "repository_dispatch" ] && [ "$ACTION" = "pretalx-change" ]; then @@ -89,32 +89,22 @@ jobs: esac elif [ "$EVENT_NAME" = "repository_dispatch" ]; then case "$PAYLOAD_SOURCE" in - developers-events|psf-trademarks|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;; + developers-events|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;; *) echo "::error title=Desk leads::registry-change names no known source"; exit 1 ;; esac elif [ "${CHOSEN:-all}" = "all" ]; then - SELECTED="pretalx developers-events psf-trademarks pycon-org" - elif [ "$CHOSEN" = "registry" ]; then - SELECTED="psf-trademarks pycon-org" + SELECTED="pretalx developers-events pycon-org" else SELECTED="$CHOSEN" fi echo "Sources: $SELECTED" - for source in pretalx developers-events; do + for source in pretalx developers-events pycon-org; do case " $SELECTED " in *" $source "*) RUN=true ;; *) RUN=false ;; esac echo "${source//-/_}=$RUN" >> "$GITHUB_OUTPUT" done - # The registries run in one step, each on its own - REGISTRIES="" - for registry in psf-trademarks pycon-org; do - case " $SELECTED " in - *" $registry "*) REGISTRIES="$REGISTRIES $registry" ;; - esac - done - echo "registries=${REGISTRIES# }" >> "$GITHUB_OUTPUT" echo "page=$PAGE" >> "$GITHUB_OUTPUT" # A source that fails gets a warning; the other sources still run. @@ -156,22 +146,18 @@ jobs: echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT" echo "status=$STATUS" >> "$GITHUB_OUTPUT" - - name: PyCon registries - id: registry - if: steps.plan.outputs.registries != '' - env: - REGISTRIES: ${{ steps.plan.outputs.registries }} + - name: pycon.org + id: pycon_org + if: steps.plan.outputs.pycon_org == 'true' run: | set -uo pipefail touch "$RUNNER_TEMP/desk-records.jsonl" BEFORE=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl") STATUS=ok - for registry in $REGISTRIES; do - if ! pixi run python utils/desk_leads.py "$registry" --out "$RUNNER_TEMP/desk-records.jsonl"; then - STATUS=failed - echo "::warning title=Desk leads::$registry failed; what it found is still queued" - fi - done + if ! pixi run python utils/desk_leads.py pycon-org --out "$RUNNER_TEMP/desk-records.jsonl"; then + STATUS=failed + echo "::warning title=Desk leads::pycon-org failed; what it found is still queued" + fi AFTER=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl") echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT" echo "status=$STATUS" >> "$GITHUB_OUTPUT" @@ -183,7 +169,7 @@ jobs: uses: ./.github/actions/desk-queue-write with: records: ${{ runner.temp }}/desk-records.jsonl - message: Queue leads from pretalx, developers.events and the PyCon registries + message: Queue leads from pretalx, developers.events and pycon.org - name: Summary if: ${{ !cancelled() }} @@ -192,8 +178,8 @@ jobs: PRETALX_STATUS: ${{ steps.pretalx.outputs.status }} DEVELOPERS_EVENTS_COUNT: ${{ steps.developers_events.outputs.count }} DEVELOPERS_EVENTS_STATUS: ${{ steps.developers_events.outputs.status }} - REGISTRY_COUNT: ${{ steps.registry.outputs.count }} - REGISTRY_STATUS: ${{ steps.registry.outputs.status }} + PYCON_ORG_COUNT: ${{ steps.pycon_org.outputs.count }} + PYCON_ORG_STATUS: ${{ steps.pycon_org.outputs.status }} ADDED: ${{ steps.queue.outputs.added }} run: | row() { @@ -210,7 +196,7 @@ jobs: echo "| --- | --- | --- |" row pretalx "$PRETALX_COUNT" "$PRETALX_STATUS" row developers-events "$DEVELOPERS_EVENTS_COUNT" "$DEVELOPERS_EVENTS_STATUS" - row registry "$REGISTRY_COUNT" "$REGISTRY_STATUS" + row pycon-org "$PYCON_ORG_COUNT" "$PYCON_ORG_STATUS" echo "" echo "- New in the desk queue: ${ADDED:-0}" } >> "$GITHUB_STEP_SUMMARY" @@ -219,7 +205,7 @@ jobs: - name: Fail when a source failed if: ${{ !cancelled() }} env: - STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.registry.outputs.status }} + STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.pycon_org.outputs.status }} run: | case " $STATUSES " in *" failed "*) echo "::error title=Desk leads::a source failed, see the warnings above"; exit 1 ;; diff --git a/tests/test_check_conference_update.py b/tests/test_check_conference_update.py index 2d87993e33..0cef3c2694 100644 --- a/tests/test_check_conference_update.py +++ b/tests/test_check_conference_update.py @@ -43,8 +43,15 @@ def run(action, payload): monkeypatch.setenv("RECORDS", str(records)) monkeypatch.setenv("GITHUB_OUTPUT", str(output)) monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) - runpy.run_path(str(script), run_name="__main__") - return json.loads(records.read_text(encoding="utf-8")), output.read_text(encoding="utf-8") + monkeypatch.setenv("RUNNER_TEMP", str(tmp_path)) + code = 0 + try: + runpy.run_path(str(script), run_name="__main__") + except SystemExit as exit_: + code = exit_.code or 0 + assert code == 0 + record = json.loads(records.read_text(encoding="utf-8")) if records.exists() else None + return record, output.read_text(encoding="utf-8") return run @@ -67,27 +74,28 @@ def test_conference_change_queues_a_signal_with_the_fetched_url(self, dispatch): assert "registry" not in record["payload"] assert "series=PyCon Ireland" in output - def test_listing_change_queues_a_registry_item(self, dispatch): + def test_listing_change_hands_its_lines_to_the_data_check(self, dispatch, tmp_path): listing = { "url": "https://pydata.org/upcoming-events/", "title": "PyData Upcoming", "watch_uuid": "abc", - "diff": "+ PyData Lagos 2027", - } - record, _ = dispatch("listing-change", listing) - assert record["source"] == "changedetection" - assert record["kind"] == "registry" - assert record["payload"] == { - "diff": "+ PyData Lagos 2027", - "watch_uuid": "abc", - "trigger": "website_change", - "registry": "listing", + "diff": "PyData Lagos 2027\nPyData Amsterdam", } + record, output = dispatch("listing-change", listing) + assert record is None # the next step writes the records, one per line it keeps + assert "listing=true" in output.splitlines() + assert (tmp_path / "listing-lines.txt").read_text(encoding="utf-8") == "PyData Lagos 2027\nPyData Amsterdam\n" + assert (tmp_path / "listing-url.txt").read_text(encoding="utf-8") == "https://pydata.org/upcoming-events/" + + def test_a_signal_is_a_valid_queue_record(self, dispatch): + record, output = dispatch("conference-change", PAYLOAD) + assert normalise_record(record)["kind"] == "signal" + assert "listing=true" not in output - @pytest.mark.parametrize("action", ["conference-change", "listing-change"]) - def test_the_queue_accepts_both(self, dispatch, action): - record, _ = dispatch(action, PAYLOAD) - assert normalise_record(record)["kind"] == record["kind"] + def test_the_listing_steps_run_only_for_a_listing(self): + steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["queue"]["steps"] + guarded = [step["name"] for step in steps if step.get("if") == "steps.record.outputs.listing == 'true'"] + assert guarded == ["Setup Pixi", "Check the listing lines against the data"] def test_both_dispatch_types_trigger_the_workflow(self): # PyYAML reads the bare key `on` as True diff --git a/tests/test_desk_leads.py b/tests/test_desk_leads.py index b2b189584f..1e99fc46a5 100644 --- a/tests/test_desk_leads.py +++ b/tests/test_desk_leads.py @@ -1,4 +1,4 @@ -"""Tests for desk_leads: lead importers for pretalx, developers.events and the PyCon registries. +"""Tests for desk_leads: lead importers for pretalx, developers.events, pycon.org and listing pages. Nothing here touches the network: every request goes to the responses mock, and opening a socket fails the test. @@ -38,7 +38,6 @@ from desk_leads import pretalx_slug from desk_leads import pycon_org_record from desk_leads import registry_paths -from desk_leads import trademark_items from desk_queue import add_items from desk_queue import normalise_record from desk_worklist import build_worklist @@ -731,22 +730,6 @@ def test_whole_source_failure_exits_1(self, tmp_path, web, capsys): assert "503" in capsys.readouterr().err -PSF_CONTENT = """\ -

PyCon Conferences around the World

-

The following are national-level and regional authorized PyCon conferences:

-
-
    -
  • "Kiwi PyCon", "PyCon NZ" in New Zealand
    • -
  • "PyCon MEA (Middle East & Africa)" in Dubai
    • -
  • "PyCon NP", "PyCon Nepal" in Nepal
    • -
  • "PyCon SK", "PyCon Slovakia" in Slovakia
    • -
  • "PyCon VE", "PyCon Venezuela" in Venezuela
    • -
-
-

If we have forgotten a national or regional PyCon conference in this list, please write to us.

-
  • "Not a PyCon" in a later list
-""" - REGISTRY_FILES = { "/_national/pycon-ireland.yml": "---\nname: PyCon Ireland\nflag: ie\nlocation: Ireland\nwebsite: https://python.ie\n---\n", "/_national/pycon-dk.yml": ( @@ -764,11 +747,10 @@ def test_whole_source_failure_exits_1(self, tmp_path, web, capsys): } -def registry_web(web, psf_status=200, broken=None): - if psf_status == 200: - web.add(responses.GET, desk_leads.PSF_TRADEMARKS_API, json={"content": PSF_CONTENT, "path": "/psf/pycon/"}) - else: - web.add(responses.GET, desk_leads.PSF_TRADEMARKS_API, status=psf_status) +def registry_web(web, listing_status=200, broken=None): + if listing_status != 200: + web.add(responses.GET, desk_leads.PYCON_ORG_FILES, status=listing_status) + return listing = [{"name": path, "hash": "x", "size": 90} for path in REGISTRY_FILES] listing += [{"name": "/index.html", "hash": "x", "size": 9}, {"name": "/_national/README.md", "hash": "x"}] web.add(responses.GET, desk_leads.PYCON_ORG_FILES, json={"type": "gh", "files": listing}) @@ -780,20 +762,7 @@ def registry_web(web, psf_status=200, broken=None): web.add(responses.GET, url, body=text) -class TestRegistry: - def test_trademark_items(self): - assert trademark_items(PSF_CONTENT) == [ - ["Kiwi PyCon", "PyCon NZ"], - ["PyCon MEA (Middle East & Africa)"], - ["PyCon NP", "PyCon Nepal"], - ["PyCon SK", "PyCon Slovakia"], - ["PyCon VE", "PyCon Venezuela"], - ] - - def test_trademark_page_without_the_list(self): - with pytest.raises(ValueError, match="no list"): - trademark_items("

Nothing here

") - +class TestPyconOrg: def test_front_matter(self): assert front_matter(REGISTRY_FILES["/_regional/pygotham.yml"]) == { "name": "PyGotham", @@ -832,53 +801,20 @@ def test_registry_paths(self): with pytest.raises(ValueError, match="no files"): registry_paths({"files": None}) - def test_both_registries(self, index, web): + def test_untracked_events(self, index, web): registry_web(web) - records = desk_leads.registry_leads(index) - assert [(record["payload"]["registry"], record["payload"]["names"]) for record in records] == [ - ("psf-trademarks", ["PyCon NP", "PyCon Nepal"]), - ("psf-trademarks", ["PyCon VE", "PyCon Venezuela"]), - ("pycon.org", ["PyGotham"]), - ] - assert records[0] == { - "source": "registry", - "kind": "registry", - "title": "PyCon NP", - "url": "https://www.python.org/psf/trademarks/pycon/", - "year": None, - "payload": { - "names": ["PyCon NP", "PyCon Nepal"], - "change": "untracked", - "registry": "psf-trademarks", - "evidence_url": "https://www.python.org/psf/trademarks/pycon/", - }, - } + records = desk_leads.collect("pycon-org", dataset(), index, TODAY) + assert [record["title"] for record in records] == ["PyGotham"] def test_one_failed_file_is_skipped(self, index, web, capsys): registry_web(web, broken="/_regional/pygotham.yml") - records = desk_leads.registry_leads(index) - assert [record["title"] for record in records] == ["PyCon NP", "PyCon VE"] + assert desk_leads.pycon_org_leads(index) == [] assert "pygotham.yml" in capsys.readouterr().err - def test_failed_registry_keeps_the_other(self, index, web): - registry_web(web, psf_status=500) - with pytest.raises(SourceError, match="psf-trademarks") as raised: - desk_leads.registry_leads(index) - assert [record["title"] for record in raised.value.records] == ["PyGotham"] - - @pytest.mark.parametrize( - ("source", "titles"), - [("psf-trademarks", ["PyCon NP", "PyCon VE"]), ("pycon-org", ["PyGotham"])], - ) - def test_each_registry_on_its_own(self, index, web, source, titles): - registry_web(web) - records = desk_leads.collect(source, dataset(), index, TODAY) - assert [record["title"] for record in records] == titles - - def test_one_registry_failing_alone_is_an_error(self, index, web): - registry_web(web, psf_status=500) - with pytest.raises(SourceError, match="psf-trademarks") as raised: - desk_leads.collect("psf-trademarks", dataset(), index, TODAY) + def test_failed_listing_is_an_error(self, index, web): + registry_web(web, listing_status=500) + with pytest.raises(SourceError, match="500 Server Error") as raised: + desk_leads.collect("pycon-org", dataset(), index, TODAY) assert raised.value.records == [] @@ -897,6 +833,73 @@ def pretalx_web(web): not_found(web, "pycongr27", *OTHER_PROBES) +LISTING_URL = "https://pydata.org/upcoming-events/" +LISTING_LINES = [ + "(added) SciPy India 2026", + "SciPy India 2027", + "PyData Lagos 2027", + "", + "+ PyData Berlin", + "PyData Lagos 2027", + '"Kiwi PyCon", "PyCon NZ" in New Zealand', + "“PyCon NP”, “PyCon Nepal” in Nepal", + "Brand New Conference", +] + + +class TestListing: + def test_names_are_the_quoted_ones_or_the_line(self): + assert desk_leads.listing_names('"PyCon AT", "PyCon Austria" in Austria') == ["PyCon AT", "PyCon Austria"] + assert desk_leads.listing_names("“PyCon NP”, “PyCon Nepal” in Nepal") == ["PyCon NP", "PyCon Nepal"] + assert desk_leads.listing_names(" PyData Amsterdam ") == ["PyData Amsterdam"] + + def test_drops_what_the_data_tracks_and_routes_the_rest(self, index): + records = desk_leads.listing_records(LISTING_LINES, LISTING_URL, index) + assert [(r["title"], r["year"], r["payload"]["classification"]) for r in records] == [ + ("SciPy India 2027", 2027, "MISSING_EDITION"), + ("PyData Lagos 2027", 2027, "NEW_SERIES"), + ("“PyCon NP”, “PyCon Nepal” in Nepal", None, "NEW_SERIES"), + ("Brand New Conference", None, "NEW_SERIES"), + ] + + def test_record_shape(self, index): + record = desk_leads.listing_records(["SciPy India 2027"], LISTING_URL, index)[0] + assert record["source"] == "changedetection" + assert record["kind"] == "candidate" + assert record["url"] == LISTING_URL + assert record["payload"]["flags"] == ["listing"] + assert record["payload"]["names"] == ["SciPy India 2027"] + assert record["payload"]["fields"] == {"conference": "SciPy India", "year": 2027} + assert record["payload"]["matched"]["conference"] == "SciPy India" + assert normalise_record(record, NOW)["kind"] == "candidate" + + def test_cli_reads_lines_and_writes_records(self, tmp_path, capsys): + write_data(tmp_path) + lines = tmp_path / "lines.txt" + lines.write_text("\n".join(LISTING_LINES) + "\n", encoding="utf-8") + out = tmp_path / "records.jsonl" + args = ["listing", "--base", str(tmp_path), "--lines", str(lines), "--url", LISTING_URL, "--out", str(out)] + assert main(args) == 0 + assert len(out.read_text(encoding="utf-8").splitlines()) == 4 + assert "desk_leads: listing: 4 record(s)" in capsys.readouterr().err + + def test_records_reach_their_skills(self, tmp_path, index): + data = dataset() + records = desk_leads.listing_records(LISTING_LINES, LISTING_URL, index) + assert {result.get("status") for result in add_items(tmp_path, records, NOW)} == {"added"} + worklist = build_worklist(data, load_queue(tmp_path), TODAY, scout=False) + skills = {item["conference"]: item["skill"] for item in worklist["items"] if item["queue_refs"]} + assert skills["SciPy India"] == "new-year-checker" + assert skills["PyData Lagos"] == "scouter" + assert set(skills.values()) <= {"new-year-checker", "scouter"} + + def test_cli_listing_needs_lines_and_url(self, tmp_path): + with pytest.raises(SystemExit): + main(["listing", "--base", str(tmp_path), "--url", LISTING_URL]) + with pytest.raises(SystemExit): + main(["pretalx", "--base", str(tmp_path), "--lines", str(tmp_path / "x")]) + + class TestCli: def test_appends_json_lines_and_counts(self, tmp_path, web, sleeps, capsys): write_data(tmp_path) @@ -959,14 +962,19 @@ def test_url_off_pretalx_checks_nothing(self, tmp_path, web, capsys): def test_url_needs_the_pretalx_source(self, tmp_path): with pytest.raises(SystemExit): - main(["registry", "--base", str(tmp_path), "--url", "https://pretalx.com/x/cfp"]) + main(["pycon-org", "--base", str(tmp_path), "--url", "https://pretalx.com/x/cfp"]) + + @pytest.mark.parametrize("source", ["registry", "psf-trademarks"]) + def test_retired_sources_are_refused(self, tmp_path, source): + with pytest.raises(SystemExit): + main([source, "--base", str(tmp_path)]) - def test_failed_registry_still_writes_the_other(self, tmp_path, web, capsys): + def test_failed_source_exits_1(self, tmp_path, web, capsys): write_data(tmp_path) - registry_web(web, psf_status=500) + registry_web(web, listing_status=500) out = tmp_path / "records.jsonl" - assert main(["registry", "--base", str(tmp_path), "--out", str(out)]) == 1 - assert [json.loads(line)["title"] for line in out.read_text(encoding="utf-8").splitlines()] == ["PyGotham"] + assert main(["pycon-org", "--base", str(tmp_path), "--out", str(out)]) == 1 + assert "pycon-org: 0 record(s)" in capsys.readouterr().err class TestQueueRoundTrip: @@ -979,8 +987,7 @@ def test_records_route_to_their_skills(self, tmp_path, web, sleeps): data = dataset() index = TrackingIndex.from_data(data) records = [] - # "registry" reads both registries, so their own sources would only repeat it - for source in ("pretalx", "developers-events", "registry"): + for source in ("pretalx", "developers-events", "pycon-org"): records.extend(desk_leads.collect(source, data, index, TODAY)) for record in records: @@ -1005,8 +1012,6 @@ def test_records_route_to_their_skills(self, tmp_path, web, sleeps): (("candidate",), "PyCon Greece", 2027): "new-year-checker", (("candidate",), "PyData Lagos", 2027): "scouter", (("candidate",), "Python Hellas Summit", 2027): "scouter", - (("registry",), "PyCon Nepal", None): "scouter", - (("registry",), "PyCon Venezuela", None): "scouter", (("registry",), "PyGotham", None): "scouter", } diff --git a/utils/desk_leads.py b/utils/desk_leads.py index 65421f63fb..20e1d9e4e2 100644 --- a/utils/desk_leads.py +++ b/utils/desk_leads.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Find leads for the deadline desk on pretalx, developers.events and the PyCon registries. +"""Find leads for the deadline desk on pretalx, developers.events and the pycon.org registry. Each source writes queue records as JSON lines, which utils/desk_queue.py adds to the desk-queue branch. Nothing here writes the data files: the desk checks @@ -12,9 +12,12 @@ robots.txt of pretalx.com disallows ``/api/``. - ``developers-events``: upcoming Python events on developers.events that the data does not track (kind ``candidate``). -- ``registry``: PyCon names on the PSF trademark list, and events in the - pycon.org registry, that the data does not track (kind ``registry``). - ``psf-trademarks`` and ``pycon-org`` read one of the two each. +- ``pycon-org``: events in the pycon.org registry that the data does not + track (kind ``registry``). +- ``listing``: the lines a changedetection watch on a page that lists many + events saw added, one event per line (kind ``candidate``, flag ``listing``), + for the editions and series the data does not track. ``--lines`` names the + file, ``--url`` the page. The PSF PyCon trademark list comes this way. A find is tracked when it is the same series as an entry in any of the three data files, by the rules in AGENTS.md "Names": its name (year and a trailing @@ -33,11 +36,10 @@ python utils/desk_leads.py pretalx --out records.jsonl python utils/desk_leads.py developers-events --today 2026-10-14 - python utils/desk_leads.py registry --base . --out records.jsonl + python utils/desk_leads.py pycon-org --base . --out records.jsonl """ import argparse -import html import json import re import sys @@ -71,9 +73,7 @@ from .sort_yaml import DATEFORMAT from .tidy_conf.yaml import load_title_mappings -SOURCES = ("pretalx", "developers-events", "registry", "psf-trademarks", "pycon-org") -# The two registries "registry" reads together; each also runs on its own -REGISTRIES = ("psf-trademarks", "pycon-org") +SOURCES = ("pretalx", "developers-events", "pycon-org") DATA_FILES = ("conferences", "archive", "legacy") USER_AGENT = "pythondeadlin.es desk leads (+https://pythondeadlin.es)" TIMEOUT = 20 @@ -93,9 +93,6 @@ PRETALX_CFP = "https://pretalx.com/{slug}/cfp" DEVELOPERS_EVENTS = "https://developers.events/all-events.json" -PSF_TRADEMARKS = "https://www.python.org/psf/trademarks/pycon/" -PSF_TRADEMARKS_API = "https://www.python.org/api/v1/pages/page/852/?format=json" -PSF_LIST_MARKER = "national-level and regional authorized PyCon conferences" PYCON_ORG_FILES = "https://data.jsdelivr.com/v1/packages/gh/PyCon/pycon.org@master?structure=flat" PYCON_ORG_RAW = "https://raw.githubusercontent.com/PyCon/pycon.org/master" @@ -1093,97 +1090,111 @@ def developers_events_records(events: Iterable[Any], index: TrackingIndex, today return records -def developers_events_leads(index: TrackingIndex, today: date) -> list[dict]: - """Fetch developers.events and return its candidate records. +def listing_names(line: str) -> list[str]: + """Return the event names on one line of a watched listing page. Parameters ---------- - index : TrackingIndex - The data's series. - today : date - The date the run is for. + line : str + One line, e.g. "PyData Amsterdam" or ``"PyCon AT", "PyCon Austria" in Austria``. Returns ------- - list[dict] - Candidate records. - - Raises - ------ - SourceError - When all-events.json cannot be read. + list[str] + The quoted names when the line quotes any, else the whole line. """ - events = fetch_json(DEVELOPERS_EVENTS) - if not isinstance(events, list): - raise SourceError(f"{DEVELOPERS_EVENTS}: not a list of events") - return developers_events_records(events, index, today) + quoted = [name.strip() for name in re.findall(r"[\"“]([^\"“”]+)[\"”]", line) if name.strip()] + return quoted or [line.strip()] + +def listing_records(lines: Iterable[str], page_url: str, index: TrackingIndex) -> list[dict]: + """Turn the lines a watched listing page added into candidate records. -def trademark_items(content: str) -> list[list[str]]: - """Read the authorised PyCon names from the PSF trademark page. + A changedetection watch on a page that lists many events keeps one event + per line, so each added line is one find. A line with a year is checked + edition by edition; a line without one (a series registry, or a list + that names no years) only says whether the series exists. Parameters ---------- - content : str - The page HTML. + lines : Iterable[str] + The added lines. + page_url : str + The watched page, which is where each find came from. + index : TrackingIndex + The data's series. Returns ------- - list[list[str]] - The quoted names of each list item after "national-level and regional - authorized PyCon conferences", in page order. - - Raises - ------ - ValueError - When the page has no such list. + list[dict] + One record per line whose series, or whose edition for the line's + year, the data does not track, in line order. """ - start = content.find(PSF_LIST_MARKER) - list_start = content.find("= 0 else -1 - list_end = content.find("", list_start) if list_start >= 0 else -1 - if list_end < 0: - raise ValueError("no list of authorised PyCon conferences") - items = [] - for raw in re.findall(r"]*>(.*?)", content[list_start:list_end], re.DOTALL): - text = re.sub(r"<[^>]+>", "", html.unescape(raw)) - names = [" ".join(name.split()) for name in re.findall(r'"([^"]+)"', text)] - if names: - items.append(names) - return items - - -def psf_records(items: Iterable[list[str]], index: TrackingIndex) -> list[dict]: - """Return a registry record for each trademark list item the data does not track. + records = [] + seen: set[str] = set() + for raw in lines: + # diff_added_clean has no markers; strip them anyway in case a body sends diff_added + line = re.sub(r"^(?:\(added\)|\+)\s*", "", " ".join(raw.split())) + if not line or line in seen: + continue + seen.add(line) + names = listing_names(line) + match = NAME_YEAR.search(line) + year = int(match[0]) if match else None + if year is None: + if any(index.is_tracked_name(name) for name in names): + continue + found = {"classification": "NEW_SERIES", "matched": None} + else: + results = [index.classify(name, year, None) for name in names] + if any(result is None for result in results): + continue + found = next((r for r in results if r["classification"] != "NEW_SERIES"), results[0]) + records.append( + { + "source": "changedetection", + "kind": "candidate", + "title": line[:200], + "url": page_url, + "year": year, + "payload": { + "classification": found["classification"], + "fields": {"conference": clean_name(names[0]), "year": year}, + "source_id": f"{page_url}#{line}", + "evidence_url": page_url, + "matched": found["matched"], + "flags": ["listing"], + "names": names, + }, + }, + ) + return records + + +def developers_events_leads(index: TrackingIndex, today: date) -> list[dict]: + """Fetch developers.events and return its candidate records. Parameters ---------- - items : Iterable[list[str]] - Names per list item, from trademark_items. index : TrackingIndex The data's series. + today : date + The date the run is for. Returns ------- list[dict] - Records for items none of whose names is tracked. + Candidate records. + + Raises + ------ + SourceError + When all-events.json cannot be read. """ - return [ - { - "source": "registry", - "kind": "registry", - "title": names[0], - "url": PSF_TRADEMARKS, - "year": None, - "payload": { - "names": list(names), - "change": "untracked", - "registry": "psf-trademarks", - "evidence_url": PSF_TRADEMARKS, - }, - } - for names in items - if names and not any(index.is_tracked_name(name) for name in names) - ] + events = fetch_json(DEVELOPERS_EVENTS) + if not isinstance(events, list): + raise SourceError(f"{DEVELOPERS_EVENTS}: not a list of events") + return developers_events_records(events, index, today) def front_matter(text: str) -> dict | None: @@ -1276,63 +1287,45 @@ def pycon_org_record(meta: dict, raw_url: str, index: TrackingIndex) -> dict | N } -def registry_leads(index: TrackingIndex, parts: Iterable[str] = REGISTRIES) -> list[dict]: - """Read the PSF trademark list, the pycon.org registry, or both. +def pycon_org_leads(index: TrackingIndex) -> list[dict]: + """Read the pycon.org registry. Parameters ---------- index : TrackingIndex The data's series. - parts : Iterable[str], optional - Which registries to read, of REGISTRIES; by default both. Returns ------- list[dict] - Registry records: the trademark list's, then pycon.org's. + Registry records for the events the data does not track. Raises ------ SourceError - When a registry cannot be read, carrying the other's records. + When the registry's file listing cannot be read. """ - parts = set(parts) - records: list[dict] = [] - errors = [] - if "psf-trademarks" in parts: - try: - page = fetch_json(PSF_TRADEMARKS_API) - content = page.get("content") if isinstance(page, dict) else None - if not isinstance(content, str): - raise ValueError("no content in the page JSON") - records.extend(psf_records(trademark_items(content), index)) - except (SourceError, ValueError) as error: - errors.append(f"psf-trademarks: {error}") - - paths = [] - if "pycon-org" in parts: - try: - paths = registry_paths(fetch_json(PYCON_ORG_FILES)) - except (SourceError, ValueError) as error: - errors.append(f"pycon.org: {error}") + listing = fetch_json(PYCON_ORG_FILES) + try: + paths = registry_paths(listing) + except ValueError as error: + raise SourceError(f"{PYCON_ORG_FILES}: {error}") from error + records = [] for path in paths: raw_url = f"{PYCON_ORG_RAW}{path}" try: response = fetch(raw_url) response.raise_for_status() except requests.RequestException as error: - report(f"registry: {raw_url}: {first_line(error)}, skipped") + report(f"pycon-org: {raw_url}: {first_line(error)}, skipped") continue meta = front_matter(response.text) if meta is None: - report(f"registry: {raw_url} has no front matter, skipped") + report(f"pycon-org: {raw_url} has no front matter, skipped") continue record = pycon_org_record(meta, raw_url, index) if record is not None: records.append(record) - - if errors: - raise SourceError("; ".join(errors), records) return records @@ -1364,10 +1357,8 @@ def collect(source: str, data: dict[str, list[dict]], index: TrackingIndex, toda records = pretalx_leads(data, index, today) elif source == "developers-events": records = developers_events_leads(index, today) - elif source == "registry": - records = registry_leads(index) - elif source in REGISTRIES: - records = registry_leads(index, (source,)) + elif source == "pycon-org": + records = pycon_org_leads(index) else: raise ValueError(f"unknown source {source!r}") return _unique(records) @@ -1419,7 +1410,8 @@ def main(argv: list[str] | None = None) -> int: before the failure are still written. """ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("source", choices=SOURCES) + # listing reads the lines a watch saw added, not a source of its own, so collect() leaves it out + parser.add_argument("source", choices=(*SOURCES, "listing")) parser.add_argument("--base", type=Path, default=Path(), help="Repository root (default: current directory)") parser.add_argument( "--today", @@ -1429,14 +1421,32 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--out", type=Path, help="File to append the records to (default: stdout)") parser.add_argument( "--url", - help="pretalx only: check just the entries on this CfP page, which may carry a watch's year template", + help=( + "pretalx: check just the entries on this CfP page, which may carry a watch's year template; " + "listing: the watched page the lines came from" + ), ) + parser.add_argument("--lines", type=Path, help="listing only: file with the lines the watched page added") args = parser.parse_args(argv) + if args.source == "listing": + if args.lines is None or args.url is None: + parser.error("the listing source needs --lines and --url") + try: + lines = args.lines.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as error: + parser.error(f"cannot read {args.lines}: {error}") + records = listing_records(lines, args.url, TrackingIndex.from_data(load_data(args.base))) + write_records(records, args.out) + print(f"desk_leads: listing: {len(records)} record(s)", file=sys.stderr) + return 0 + if args.lines is not None: + parser.error("--lines works with the listing source only") + slug = None if args.url is not None: if args.source != "pretalx": - parser.error("--url works with the pretalx source only") + parser.error("--url works with the pretalx and listing sources only") slug = pretalx_slug(resolve_watch_url(args.url)) if slug is None: print(f"desk_leads: pretalx: {args.url} is not a pretalx.com event page, nothing to check", file=sys.stderr) diff --git a/utils/desk_queue.py b/utils/desk_queue.py index d8077ac51c..467d5d5cda 100644 --- a/utils/desk_queue.py +++ b/utils/desk_queue.py @@ -139,8 +139,8 @@ | `pretalx` | `drift` | `conference`, `year`, `diffs` | | `pretalx`, `developers-events`, `canonical-import` | `candidate` | `classification`, `fields`, `source_id`, \ `evidence_url`, `matched`, `flags` | +| `changedetection` | `candidate` | as above, with `names` and the flag `listing` | | `registry` | `registry` | `names`, `change`, `registry`, `evidence_url` | -| `changedetection` | `registry` | `registry` (`listing`), `diff`, `watch_uuid`, `trigger` | A `drift` diff maps a field to `{"data", "source"}`: the value in the data and what the source states. A candidate's `classification` is @@ -149,9 +149,11 @@ `flags` say), and `matched` names the entry it matched, or is `null`. The worklist routes a candidate by its own match against the data, not by its classification. A `lead-only` flag means the source's licence keeps it to -leads: use only its name, year and link. `registry` is `psf-trademarks` or -`pycon.org`, and a pycon.org item adds `website`; from changedetection it is -`listing`, a watched page that lists many events, whose `diff` holds the change. +leads: use only its name, year and link. A `listing` flag marks one line a +changedetection watch on a page that lists many events added, with `names` +the names on that line. `registry` is `pycon.org`, and the item adds +`website`; older items can say `psf-trademarks`, from before the trademark +list became a listing page. Every item is data for the desk to check on the live sources. Nothing in a queue item reaches the conference data without that check.