Check each line a listing page adds against the data

.claude/skills/scouter/SKILL.md

@@ -14,7 +14,7 @@ The desk hands you the daily `scout` item, or an item about a series no data fil
      - `lead-only`: an aggregator lead. Copy no value from it; take every value from a primary source.
      - `no-website`: the shared sheet has this edition but no website. Search for one.
      - `denylist-review: <domain>: <reason>`: the domain is on the denylist's `review` list. Copy the flag into the `flags` of the find's change set; the critic returns needs-maintainer.
-   - A `registry` item whose payload `registry` is `listing`: a watched page that lists many events changed. Each event name or link the payload's `diff` adds is a find, with the item's `url` as where it came from; fetch that page when the diff is cut short or names an event without a link. A listing page is a lead, never the evidence (step 4).
+     - `listing`: one line a watched page that lists many events added, such as "PyData Amsterdam". The item's `url` is that page and its payload `names` the names on the line. Find the event on the page or by search. A listing page is a lead, never the evidence (step 4).
    - A `maintainer-request` item: the series and URLs the comment names are the find, and the request settles its scope (step 2).
    - The `scout` item: search each of these, with `<year>` standing for this year and next (the year of the worklist's `today`, which the desk gives with the item).
      - Targeted web searches: "PyCon <year>", "PyData <year>", "Python conference call for proposals", "PyDay", "Python Pizza", framework and scientific library names with "conference", and the same in other languages ("conferencia Python", "Python Konferenz").

.github/workflows/check-conference-update.yml

@@ -6,9 +6,11 @@ name: Check Conference Update
 #
 # conference-change comes from a watch on one conference's pages and queues a
 # signal for that series. listing-change comes from a watch on a page that
-# lists many events (pydata.org/upcoming-events, barcamps.eu and the like) and
-# queues a registry item, which the worklist always gives the scouter: the
-# names and links the diff adds are the finds.
+# lists many events (pydata.org/upcoming-events, conference.scipy.org, the
+# PSF PyCon trademark list), filtered to one event per line. Each added line
+# is checked against the data in code (utils/desk_leads.py listing): a tracked
+# edition is dropped, and the rest are queued as candidates, which the
+# worklist routes to the new-year-checker or the scouter.
 
 on:
   # Triggered by changedetection.io webhook
@@ -101,16 +103,26 @@ jobs:
 
           name, year = clean_watch_title(title)
           name = " ".join(name.split())  # one line, for the step output
-          payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger}
           if listing:
-              payload["registry"] = "listing"
+              # One event per added line: the next step checks each against
+              # the data. Files, not step outputs, carry the page's text.
+              temp = os.environ["RUNNER_TEMP"]
+              with open(f"{temp}/listing-lines.txt", "w", encoding="utf-8") as f:
+                  f.write(diff if diff.endswith("\n") or not diff else diff + "\n")
+              with open(f"{temp}/listing-url.txt", "w", encoding="utf-8") as f:
+                  f.write(" ".join(resolve_watch_url(url.strip()).split()))
+              with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out:
+                  out.write(f"series={name}\nlisting=true\n")
+              print(f"Checking {len(diff.splitlines())} added line(s) from {name or title}")
+              sys.exit(0)
+          payload = {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger}
           if event_name == "workflow_dispatch":
               # The run id gives each manual request its own queue id, so
               # asking again with the same inputs is not taken as handled.
               payload["requested"] = os.environ["GITHUB_RUN_ID"]
           record = {
               "source": "changedetection",
-              "kind": "registry" if listing else "signal",
+              "kind": "signal",
               "title": title.strip(),
               # A watch that follows a series sends its URL with the year template in it
               "url": resolve_watch_url(url.strip()),
@@ -121,9 +133,24 @@ jobs:
               f.write(json.dumps(record, ensure_ascii=False) + "\n")
           with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out:
               out.write(f"series={name}\n")
-          print(f"Queueing a {trigger} {record['kind']} item for {name or title}")
+          print(f"Queueing a {trigger} signal for {name or title}")
           BUILD
 
+      - name: Setup Pixi
+        if: steps.record.outputs.listing == 'true'
+        uses: prefix-dev/setup-pixi@v0.11.2
+
+      # Each added line is one event. A line whose edition (or, without a
+      # year, whose series) the data tracks is dropped; the rest become
+      # candidates that the worklist routes like any other.
+      - name: Check the listing lines against the data
+        if: steps.record.outputs.listing == 'true'
+        run: |
+          pixi run python utils/desk_leads.py listing \
+            --lines "$RUNNER_TEMP/listing-lines.txt" \
+            --url "$(cat "$RUNNER_TEMP/listing-url.txt")" \
+            --out "$RUNNER_TEMP/desk-records.jsonl"
+
       - name: Write to the desk queue
         id: queue
         uses: ./.github/actions/desk-queue-write

.github/workflows/desk-leads.yml

@@ -1,7 +1,7 @@
 name: Desk Leads
 
-# Report-only. Looks for leads on pretalx, developers.events and the PyCon
-# registries and queues them for the desk, which checks each one on the
+# Report-only. Looks for leads on pretalx, developers.events and the pycon.org
+# registry and queues them for the desk, which checks each one on the
 # conference's own pages. Nothing here writes the conference data.
 #
 # Sources run when changedetection.io sees them change, with a slow schedule
@@ -12,19 +12,21 @@ name: Desk Leads
 #   robots.txt on pretalx.com disallows /api/. A watch on one CfP page sends
 #   pretalx-change with that page's URL, and only its entries are checked.
 #   Weekly, all of them and the next-edition guess.
-# - developers.events (CC BY-NC data, used for leads only), the PSF PyCon
-#   trademark list and the pycon.org registry: a watch on each (the Python
-#   rows of TAGS.csv, the trademark page, the registry's latest commit) sends
-#   registry-change naming its source, and that source is read in full.
-#   Monthly, all three.
+# - developers.events (CC BY-NC data, used for leads only) and the pycon.org
+#   registry: a watch on each (the Python rows of TAGS.csv, the registry's
+#   latest commit) sends registry-change naming its source, and that source
+#   is read in full. Monthly, both.
+#
+# The PSF PyCon trademark list is a listing page: its watch sends
+# listing-change to check-conference-update.yml, which checks each added line.
 #
 # A finding the desk has handled is not queued again until it changes, so a
 # lead that comes back every run adds nothing.
 
 on:
   schedule:
     - cron: '23 4 * * 1'  # Mondays 04:23 UTC: pretalx
-    - cron: '41 4 1 * *'  # the 1st of the month, 04:41 UTC: developers-events and both registries
+    - cron: '41 4 1 * *'  # the 1st of the month, 04:41 UTC: developers-events and pycon-org
   repository_dispatch:
     types: [registry-change, pretalx-change]
   workflow_dispatch:
@@ -36,8 +38,6 @@ on:
           - all
           - pretalx
           - developers-events
-          - registry
-          - psf-trademarks
           - pycon-org
         default: all
 
@@ -76,7 +76,7 @@ jobs:
           if [ "$EVENT_NAME" = "schedule" ]; then
             case "$SCHEDULE" in
               '23 4 * * 1') SELECTED="pretalx" ;;
-              '41 4 1 * *') SELECTED="developers-events psf-trademarks pycon-org" ;;
+              '41 4 1 * *') SELECTED="developers-events pycon-org" ;;
               *) echo "::error title=Desk leads::Unknown schedule $SCHEDULE"; exit 1 ;;
             esac
           elif [ "$EVENT_NAME" = "repository_dispatch" ] && [ "$ACTION" = "pretalx-change" ]; then
@@ -89,32 +89,22 @@ jobs:
             esac
           elif [ "$EVENT_NAME" = "repository_dispatch" ]; then
             case "$PAYLOAD_SOURCE" in
-              developers-events|psf-trademarks|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;;
+              developers-events|pycon-org) SELECTED="$PAYLOAD_SOURCE" ;;
               *) echo "::error title=Desk leads::registry-change names no known source"; exit 1 ;;
             esac
           elif [ "${CHOSEN:-all}" = "all" ]; then
-            SELECTED="pretalx developers-events psf-trademarks pycon-org"
-          elif [ "$CHOSEN" = "registry" ]; then
-            SELECTED="psf-trademarks pycon-org"
+            SELECTED="pretalx developers-events pycon-org"
           else
             SELECTED="$CHOSEN"
           fi
           echo "Sources: $SELECTED"
-          for source in pretalx developers-events; do
+          for source in pretalx developers-events pycon-org; do
             case " $SELECTED " in
               *" $source "*) RUN=true ;;
               *) RUN=false ;;
             esac
             echo "${source//-/_}=$RUN" >> "$GITHUB_OUTPUT"
           done
-          # The registries run in one step, each on its own
-          REGISTRIES=""
-          for registry in psf-trademarks pycon-org; do
-            case " $SELECTED " in
-              *" $registry "*) REGISTRIES="$REGISTRIES $registry" ;;
-            esac
-          done
-          echo "registries=${REGISTRIES# }" >> "$GITHUB_OUTPUT"
           echo "page=$PAGE" >> "$GITHUB_OUTPUT"
 
       # A source that fails gets a warning; the other sources still run.
@@ -156,22 +146,18 @@ jobs:
           echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT"
           echo "status=$STATUS" >> "$GITHUB_OUTPUT"
 
-      - name: PyCon registries
-        id: registry
-        if: steps.plan.outputs.registries != ''
-        env:
-          REGISTRIES: ${{ steps.plan.outputs.registries }}
+      - name: pycon.org
+        id: pycon_org
+        if: steps.plan.outputs.pycon_org == 'true'
         run: |
           set -uo pipefail
           touch "$RUNNER_TEMP/desk-records.jsonl"
           BEFORE=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl")
           STATUS=ok
-          for registry in $REGISTRIES; do
-            if ! pixi run python utils/desk_leads.py "$registry" --out "$RUNNER_TEMP/desk-records.jsonl"; then
-              STATUS=failed
-              echo "::warning title=Desk leads::$registry failed; what it found is still queued"
-            fi
-          done
+          if ! pixi run python utils/desk_leads.py pycon-org --out "$RUNNER_TEMP/desk-records.jsonl"; then
+            STATUS=failed
+            echo "::warning title=Desk leads::pycon-org failed; what it found is still queued"
+          fi
           AFTER=$(wc -l < "$RUNNER_TEMP/desk-records.jsonl")
           echo "count=$((AFTER - BEFORE))" >> "$GITHUB_OUTPUT"
           echo "status=$STATUS" >> "$GITHUB_OUTPUT"
@@ -183,7 +169,7 @@ jobs:
         uses: ./.github/actions/desk-queue-write
         with:
           records: ${{ runner.temp }}/desk-records.jsonl
-          message: Queue leads from pretalx, developers.events and the PyCon registries
+          message: Queue leads from pretalx, developers.events and pycon.org
 
       - name: Summary
         if: ${{ !cancelled() }}
@@ -192,8 +178,8 @@ jobs:
           PRETALX_STATUS: ${{ steps.pretalx.outputs.status }}
           DEVELOPERS_EVENTS_COUNT: ${{ steps.developers_events.outputs.count }}
           DEVELOPERS_EVENTS_STATUS: ${{ steps.developers_events.outputs.status }}
-          REGISTRY_COUNT: ${{ steps.registry.outputs.count }}
-          REGISTRY_STATUS: ${{ steps.registry.outputs.status }}
+          PYCON_ORG_COUNT: ${{ steps.pycon_org.outputs.count }}
+          PYCON_ORG_STATUS: ${{ steps.pycon_org.outputs.status }}
           ADDED: ${{ steps.queue.outputs.added }}
         run: |
           row() {
@@ -210,7 +196,7 @@ jobs:
             echo "| --- | --- | --- |"
             row pretalx "$PRETALX_COUNT" "$PRETALX_STATUS"
             row developers-events "$DEVELOPERS_EVENTS_COUNT" "$DEVELOPERS_EVENTS_STATUS"
-            row registry "$REGISTRY_COUNT" "$REGISTRY_STATUS"
+            row pycon-org "$PYCON_ORG_COUNT" "$PYCON_ORG_STATUS"
             echo ""
             echo "- New in the desk queue: ${ADDED:-0}"
           } >> "$GITHUB_STEP_SUMMARY"
@@ -219,7 +205,7 @@ jobs:
       - name: Fail when a source failed
         if: ${{ !cancelled() }}
         env:
-          STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.registry.outputs.status }}
+          STATUSES: ${{ steps.pretalx.outputs.status }} ${{ steps.developers_events.outputs.status }} ${{ steps.pycon_org.outputs.status }}
         run: |
           case " $STATUSES " in
             *" failed "*) echo "::error title=Desk leads::a source failed, see the warnings above"; exit 1 ;;

tests/test_check_conference_update.py

@@ -43,8 +43,15 @@ def run(action, payload):
         monkeypatch.setenv("RECORDS", str(records))
         monkeypatch.setenv("GITHUB_OUTPUT", str(output))
         monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
-        runpy.run_path(str(script), run_name="__main__")
-        return json.loads(records.read_text(encoding="utf-8")), output.read_text(encoding="utf-8")
+        monkeypatch.setenv("RUNNER_TEMP", str(tmp_path))
+        code = 0
+        try:
+            runpy.run_path(str(script), run_name="__main__")
+        except SystemExit as exit_:
+            code = exit_.code or 0
+        assert code == 0
+        record = json.loads(records.read_text(encoding="utf-8")) if records.exists() else None
+        return record, output.read_text(encoding="utf-8")
 
     return run
 
@@ -67,27 +74,28 @@ def test_conference_change_queues_a_signal_with_the_fetched_url(self, dispatch):
         assert "registry" not in record["payload"]
         assert "series=PyCon Ireland" in output
 
-    def test_listing_change_queues_a_registry_item(self, dispatch):
+    def test_listing_change_hands_its_lines_to_the_data_check(self, dispatch, tmp_path):
         listing = {
             "url": "https://pydata.org/upcoming-events/",
             "title": "PyData Upcoming",
             "watch_uuid": "abc",
-            "diff": "+ PyData Lagos 2027",
-        }
-        record, _ = dispatch("listing-change", listing)
-        assert record["source"] == "changedetection"
-        assert record["kind"] == "registry"
-        assert record["payload"] == {
-            "diff": "+ PyData Lagos 2027",
-            "watch_uuid": "abc",
-            "trigger": "website_change",
-            "registry": "listing",
+            "diff": "PyData Lagos 2027\nPyData Amsterdam",
         }
+        record, output = dispatch("listing-change", listing)
+        assert record is None  # the next step writes the records, one per line it keeps
+        assert "listing=true" in output.splitlines()
+        assert (tmp_path / "listing-lines.txt").read_text(encoding="utf-8") == "PyData Lagos 2027\nPyData Amsterdam\n"
+        assert (tmp_path / "listing-url.txt").read_text(encoding="utf-8") == "https://pydata.org/upcoming-events/"
+
+    def test_a_signal_is_a_valid_queue_record(self, dispatch):
+        record, output = dispatch("conference-change", PAYLOAD)
+        assert normalise_record(record)["kind"] == "signal"
+        assert "listing=true" not in output
 
-    @pytest.mark.parametrize("action", ["conference-change", "listing-change"])
-    def test_the_queue_accepts_both(self, dispatch, action):
-        record, _ = dispatch(action, PAYLOAD)
-        assert normalise_record(record)["kind"] == record["kind"]
+    def test_the_listing_steps_run_only_for_a_listing(self):
+        steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["queue"]["steps"]
+        guarded = [step["name"] for step in steps if step.get("if") == "steps.record.outputs.listing == 'true'"]
+        assert guarded == ["Setup Pixi", "Check the listing lines against the data"]
 
     def test_both_dispatch_types_trigger_the_workflow(self):
         # PyYAML reads the bare key `on` as True