From 87b1a080d550564885dc26ec2ea10b49d6a5e08c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 27 Oct 2026 13:46:10 -0500 Subject: [PATCH] web workflow display mirror page at /cp/display/ --- shared-module/busdisplay/BusDisplay.c | 7 + .../framebufferio/FramebufferDisplay.c | 7 + supervisor/shared/tick.c | 8 + .../shared/web_workflow/display_stream.c | 555 ++++++++++++++++++ .../shared/web_workflow/display_stream.h | 49 ++ .../shared/web_workflow/static/display.html | 43 ++ .../shared/web_workflow/static/display.js | 228 +++++++ .../shared/web_workflow/static/serial.js | 16 +- supervisor/shared/web_workflow/web_workflow.c | 35 ++ supervisor/shared/web_workflow/websocket.c | 161 ++--- supervisor/shared/web_workflow/websocket.h | 24 + supervisor/supervisor.mk | 3 +- 12 files changed, 1055 insertions(+), 81 deletions(-) create mode 100644 supervisor/shared/web_workflow/display_stream.c create mode 100644 supervisor/shared/web_workflow/display_stream.h create mode 100644 supervisor/shared/web_workflow/static/display.html create mode 100644 supervisor/shared/web_workflow/static/display.js diff --git a/shared-module/busdisplay/BusDisplay.c b/shared-module/busdisplay/BusDisplay.c index 1c8b8f6d63e..61eeda2cba4 100644 --- a/shared-module/busdisplay/BusDisplay.c +++ b/shared-module/busdisplay/BusDisplay.c @@ -25,6 +25,10 @@ #include "supervisor/shared/display.h" #include "supervisor/shared/tick.h" +#if CIRCUITPY_WEB_WORKFLOW +#include "supervisor/shared/web_workflow/display_stream.h" +#endif + #if CIRCUITPY_TINYUSB #include "supervisor/usb.h" #endif @@ -256,6 +260,9 @@ static bool _refresh_area(busdisplay_busdisplay_obj_t *self, const displayio_are if (!displayio_display_core_clip_area(&self->core, area, &clipped)) { return true; } + #if CIRCUITPY_WEB_WORKFLOW + display_stream_area_updated(&self->core, &clipped); + #endif uint16_t rows_per_buffer = displayio_area_height(&clipped); uint16_t cols_per_buffer = displayio_area_width(&clipped); uint8_t pixels_per_word = (sizeof(uint32_t) * 8) / self->core.colorspace.depth; diff --git a/shared-module/framebufferio/FramebufferDisplay.c b/shared-module/framebufferio/FramebufferDisplay.c index 8116f4b0347..473b1de943d 100644 --- a/shared-module/framebufferio/FramebufferDisplay.c +++ b/shared-module/framebufferio/FramebufferDisplay.c @@ -16,6 +16,10 @@ #include "supervisor/shared/display.h" #include "supervisor/shared/tick.h" +#if CIRCUITPY_WEB_WORKFLOW +#include "supervisor/shared/web_workflow/display_stream.h" +#endif + #if CIRCUITPY_TINYUSB #include "supervisor/usb.h" #endif @@ -200,6 +204,9 @@ static bool _refresh_area(framebufferio_framebufferdisplay_obj_t *self, const di dest += rowstride; src += rowsize; } + #if CIRCUITPY_WEB_WORKFLOW + display_stream_area_updated(&self->core, &subrectangle); + #endif // Run background tasks so they can run during an explicit refresh. // Auto-refresh won't run background tasks here because it is a background task itself. RUN_BACKGROUND_TASKS; diff --git a/supervisor/shared/tick.c b/supervisor/shared/tick.c index 81815c0813b..4ae195a58a7 100644 --- a/supervisor/shared/tick.c +++ b/supervisor/shared/tick.c @@ -31,6 +31,10 @@ #include "shared-module/audiofilewriter/AudioFileWriter.h" #endif +#if CIRCUITPY_WEB_WORKFLOW +#include "supervisor/shared/web_workflow/display_stream.h" +#endif + #include "shared-bindings/microcontroller/__init__.h" #if CIRCUITPY_WATCHDOG @@ -90,6 +94,10 @@ void supervisor_tick(void) { keypad_tick(); #endif + #if CIRCUITPY_WEB_WORKFLOW + display_stream_tick(); + #endif + background_callback_add(&tick_callback, supervisor_background_tick, NULL); } diff --git a/supervisor/shared/web_workflow/display_stream.c b/supervisor/shared/web_workflow/display_stream.c new file mode 100644 index 00000000000..365b7ee1b29 --- /dev/null +++ b/supervisor/shared/web_workflow/display_stream.c @@ -0,0 +1,555 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Tim Cocks for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#include "supervisor/shared/web_workflow/display_stream.h" + +#if CIRCUITPY_WEB_WORKFLOW && CIRCUITPY_DISPLAYIO + +#include +#include + +#include "py/mperrno.h" +#include "shared-bindings/socketpool/Socket.h" +#include "shared-module/displayio/__init__.h" +#include "supervisor/port.h" +#include "supervisor/shared/tick.h" +#include "supervisor/shared/web_workflow/websocket.h" +#include "supervisor/workflow.h" + +#if CIRCUITPY_BUSDISPLAY +#include "shared-bindings/busdisplay/BusDisplay.h" +#endif +#if CIRCUITPY_FRAMEBUFFERIO +#include "shared-bindings/framebufferio/FramebufferDisplay.h" +#endif + +// Room for one message: the websocket header, the rectangle header and its pixels. +#define MESSAGE_SIZE (4096) +#if CIRCUITPY_BUSDISPLAY +// Room for rows rendered from a display without a framebuffer, and the mask +// rendering needs for them. Pixels are a byte or more there. +#define RENDER_SIZE (2048) +#define RENDER_MASK_WORDS (RENDER_SIZE / 32 + 1) +#else +#define RENDER_SIZE (0) +#define RENDER_MASK_WORDS (0) +#endif +#define BUFFER_SIZE (MESSAGE_SIZE + RENDER_SIZE + RENDER_MASK_WORDS * sizeof(uint32_t)) +#define WS_HEADER_SIZE (4) +#define RECT_HEADER_SIZE (10) +#define PAYLOAD_SIZE (MESSAGE_SIZE - WS_HEADER_SIZE) +#define MAX_ROWS (1024) +// How long one background pass may spend sending. +#define PASS_BUDGET_MS (8) +// How often the connection is checked. Each check can cost a command to the +// network co-processor, so it isn't done every pass. +#define POLL_MS (250) +// Passes that send start at least this far apart, so the stream takes at most +// PASS_BUDGET_MS of every PASS_INTERVAL_MS however fast the display changes or +// however slowly the client reads. Changes in between are sent together. +#define PASS_INTERVAL_MS (20) + +#define ENCODING_RAW (0) +#define ENCODING_RLE (1) + +static websocket_t ws; +// Set while a client is connected and the message buffer is allocated. +static bool streaming; +static uint8_t *message; +// The part of message still to send. +static size_t message_start; +static size_t message_end; +#if CIRCUITPY_BUSDISPLAY +// In the same allocation, after message. +static uint32_t *render_buffer; +static uint32_t *render_mask; +#endif + +// The display being streamed and what the client was told about it. target_fb +// is set when it's read from a framebuffer, otherwise rows are rendered. +static displayio_display_core_t *target; +#if CIRCUITPY_FRAMEBUFFERIO +static framebufferio_framebufferdisplay_obj_t *target_fb; +#endif +static const char *format; +static uint16_t width; +static uint16_t height; +static uint8_t depth; +static void *pixels; +static bool hello_needed; +// When the connection was last checked. +static uint64_t last_poll_ms; +// When the last pass that sent started. +static uint64_t last_pass_ms; +// Ticks left until the next pass is requested, and whether the tick is enabled for it. +static volatile uint8_t pass_countdown; +static bool tick_enabled; + +// Rows written since they were last sent, and the columns written in any of them. +static uint8_t dirty_rows[MAX_ROWS / 8]; +static uint16_t dirty_x1; +static uint16_t dirty_x2; +// Where the search for dirty rows continues, so the top doesn't starve the bottom. +static uint16_t next_row; + +void display_stream_init(void) { + socketpool_socket_reset(&ws.socket); +} + +static void _mark_dirty(uint16_t x1, uint16_t y1, uint16_t x2, uint16_t y2) { + for (uint16_t y = y1; y < y2 && y < MAX_ROWS; y++) { + dirty_rows[y / 8] |= 1 << (y % 8); + } + dirty_x1 = MIN(dirty_x1, x1); + dirty_x2 = MAX(dirty_x2, x2); +} + +static void _mark_all_dirty(void) { + _mark_dirty(0, 0, width, height); +} + +// Some ports only run the web workflow when a socket has data, so the tick asks +// for the next pass, PASS_INTERVAL_MS after the last one. +static void _pass_later(void) { + if (pass_countdown > 0) { + return; + } + if (!tick_enabled) { + supervisor_enable_tick(); + tick_enabled = true; + } + uint64_t since = supervisor_ticks_ms64() - last_pass_ms; + pass_countdown = since >= PASS_INTERVAL_MS ? 1 : PASS_INTERVAL_MS - since; +} + +static void _no_pass_needed(void) { + pass_countdown = 0; + if (tick_enabled) { + supervisor_disable_tick(); + tick_enabled = false; + } +} + +void display_stream_tick(void) { + if (pass_countdown > 0 && --pass_countdown == 0) { + supervisor_workflow_request_background(); + } +} + +static void _stop(void) { + _no_pass_needed(); + if (!common_hal_socketpool_socket_get_closed(&ws.socket)) { + common_hal_socketpool_socket_close(&ws.socket); + } + if (message != NULL) { + port_free(message); + message = NULL; + } + streaming = false; + target = NULL; + #if CIRCUITPY_FRAMEBUFFERIO + target_fb = NULL; + #endif +} + +void display_stream_handoff(socketpool_socket_obj_t *socket) { + if (streaming && message_start < message_end) { + // The close frame can't go in the middle of a message, so finish it if the + // socket takes it now. A client that stopped reading is closed without one + // rather than holding everything up. + int left = message_end - message_start; + if (socketpool_socket_send(&ws.socket, message + message_start, left) < left) { + common_hal_socketpool_socket_close(&ws.socket); + } + } + if (message != NULL) { + port_free(message); + message = NULL; + } + streaming = false; + websocket_take_socket(&ws, socket); + uint32_t *buffer = port_malloc(BUFFER_SIZE, false); + message = (uint8_t *)buffer; + if (message == NULL) { + _stop(); + return; + } + #if CIRCUITPY_BUSDISPLAY + render_buffer = buffer + MESSAGE_SIZE / sizeof(uint32_t); + render_mask = render_buffer + RENDER_SIZE / sizeof(uint32_t); + #endif + message_start = 0; + message_end = 0; + target = NULL; + #if CIRCUITPY_FRAMEBUFFERIO + target_fb = NULL; + #endif + format = NULL; + hello_needed = true; + last_poll_ms = supervisor_ticks_ms64(); + last_pass_ms = 0; + streaming = true; +} + +void display_stream_area_updated(displayio_display_core_t *core, const displayio_area_t *area) { + if (!streaming) { + return; + } + if (core == target) { + _mark_dirty(area->x1, area->y1, area->x2, area->y2); + } + // Another display may be a new one to switch to. + _pass_later(); +} + +static const char *_format_of(const _displayio_colorspace_t *colorspace) { + if (colorspace->depth < 8 && !colorspace->pixels_in_byte_share_row) { + return NULL; + } + switch (colorspace->depth) { + case 32: + return "bgrx8888"; + case 16: + return colorspace->reverse_bytes_in_word ? "rgb565be" : "rgb565le"; + case 8: + return colorspace->grayscale ? "gray8" : "rgb332"; + case 4: + return colorspace->grayscale ? "gray4" : "rgbd4"; + case 2: + return "gray2"; + case 1: + return "gray1"; + } + return NULL; +} + +// Finds the display to stream and notices when it changes. +static void _check_display(void) { + displayio_display_core_t *core = NULL; + void *fb = NULL; + void *new_pixels = NULL; + for (size_t i = 0; i < CIRCUITPY_DISPLAY_LIMIT && core == NULL; i++) { + const mp_obj_type_t *type = displays[i].display_base.type; + #if CIRCUITPY_FRAMEBUFFERIO + if (type == &framebufferio_framebufferdisplay_type) { + fb = &displays[i].framebuffer_display; + core = &displays[i].framebuffer_display.core; + new_pixels = displays[i].framebuffer_display.bufinfo.buf; + } + #endif + #if CIRCUITPY_BUSDISPLAY + if (type == &busdisplay_busdisplay_type) { + core = &displays[i].display.core; + } + #endif + (void)type; + } + const char *new_format = NULL; + uint16_t new_width = 0; + uint16_t new_height = 0; + if (core != NULL) { + // The core's area is the panel's size, whatever the rotation. + new_width = core->area.x2; + new_height = core->area.y2; + new_format = _format_of(&core->colorspace); + size_t row_bytes = (new_width * core->colorspace.depth + 7) / 8; + if (new_height > MAX_ROWS || RECT_HEADER_SIZE + row_bytes + row_bytes / 128 + 1 > PAYLOAD_SIZE) { + new_format = NULL; + } + if (fb != NULL ? new_pixels == NULL : + core->colorspace.depth < 8 || row_bytes > RENDER_SIZE) { + new_format = NULL; + } + if (new_format == NULL) { + core = NULL; + fb = NULL; + new_pixels = NULL; + } + } + if (core == target && + (core == NULL || (new_format == format && new_width == width && new_height == height && + new_pixels == pixels))) { + return; + } + target = core; + #if CIRCUITPY_FRAMEBUFFERIO + target_fb = fb; + #endif + format = new_format; + width = new_width; + height = new_height; + depth = core != NULL ? core->colorspace.depth : 0; + pixels = new_pixels; + memset(dirty_rows, 0, sizeof(dirty_rows)); + dirty_x1 = UINT16_MAX; + dirty_x2 = 0; + next_row = 0; + _mark_all_dirty(); + hello_needed = true; +} + +static void _finish_message(uint8_t opcode, size_t payload_len) { + uint8_t *payload = message + WS_HEADER_SIZE; + if (payload_len <= 125) { + message_start = WS_HEADER_SIZE - 2; + message[message_start + 1] = payload_len; + } else { + message_start = 0; + message[1] = 126; + message[2] = payload_len >> 8; + message[3] = payload_len & 0xff; + } + message[message_start] = 0x80 | opcode; + message_end = payload + payload_len - message; +} + +static void _build_hello(void) { + const _displayio_colorspace_t *colorspace = target != NULL ? &target->colorspace : NULL; + int len = snprintf((char *)message + WS_HEADER_SIZE, PAYLOAD_SIZE, + "{\"width\": %d, \"height\": %d, \"format\": \"%s\", \"lsb_first\": %s}", + width, height, format != NULL ? format : "none", + colorspace != NULL && colorspace->reverse_pixels_in_byte ? "false" : "true"); + _finish_message(0x1, len); + hello_needed = false; +} + +// Run length encodes one row of units into out. Returns the length written. +static size_t _encode_row(const uint8_t *src, size_t units, size_t unit, uint8_t *out) { + // A run of single bytes only saves space from three on. + size_t min_run = unit == 1 ? 3 : 2; + uint8_t *start = out; + size_t i = 0; + while (i < units) { + size_t run = 1; + while (i + run < units && run < 129 && memcmp(src + i * unit, src + (i + run) * unit, unit) == 0) { + run++; + } + if (run >= min_run) { + *out++ = 0x80 | (run - 2); + memcpy(out, src + i * unit, unit); + out += unit; + i += run; + continue; + } + size_t literal_start = i; + size_t count = 0; + while (i < units && count < 128) { + size_t same = 1; + while (same < min_run && i + same < units && + memcmp(src + i * unit, src + (i + same) * unit, unit) == 0) { + same++; + } + if (same >= min_run) { + break; + } + i++; + count++; + } + *out++ = count - 1; + memcpy(out, src + literal_start * unit, count * unit); + out += count * unit; + } + return out - start; +} + +static bool _row_dirty(uint16_t y) { + return dirty_rows[y / 8] & (1 << (y % 8)); +} + +// How many rows _read_rows can give at once. +static uint16_t _rows_at_once(size_t row_bytes) { + #if CIRCUITPY_FRAMEBUFFERIO + if (target_fb != NULL) { + return UINT16_MAX; + } + #endif + return RENDER_SIZE / row_bytes; +} + +// Returns rows [y, y + count) of columns [x1, x2), each stride bytes after the +// last: straight from the framebuffer, or rendered from the display's group. +static const uint8_t *_read_rows(uint16_t y, uint16_t count, uint16_t x1, uint16_t x2, size_t *stride) { + #if CIRCUITPY_FRAMEBUFFERIO + if (target_fb != NULL) { + *stride = target_fb->row_stride; + return (const uint8_t *)pixels + target_fb->first_pixel_offset + y * *stride + x1 * depth / 8; + } + #endif + #if CIRCUITPY_BUSDISPLAY + displayio_area_t area = { .x1 = x1, .y1 = y, .x2 = x2, .y2 = y + count, .next = NULL }; + *stride = (x2 - x1) * depth / 8; + memset(render_buffer, 0, count * *stride); + memset(render_mask, 0, RENDER_MASK_WORDS * sizeof(uint32_t)); + displayio_display_core_fill_area(target, &area, render_mask, render_buffer); + return (const uint8_t *)render_buffer; + #else + return NULL; + #endif +} + +// Builds a message from the next dirty rows, adding rows until it's full or +// end_ms. Returns false when there are none. +static bool _build_rect(uint64_t end_ms) { + uint16_t y = next_row; + uint16_t searched = 0; + while (searched < height && !_row_dirty(y)) { + y = y + 1 < height ? y + 1 : 0; + searched++; + } + if (searched == height) { + dirty_x1 = UINT16_MAX; + dirty_x2 = 0; + return false; + } + + uint16_t x1 = MIN(dirty_x1, width); + uint16_t x2 = MIN(dirty_x2, width); + if (depth < 8) { + // Whole bytes only. + uint16_t per_byte = 8 / depth; + x1 = x1 / per_byte * per_byte; + x2 = MIN((x2 + per_byte - 1) / per_byte * per_byte, width); + } + if (x1 >= x2) { + x1 = 0; + x2 = width; + } + + size_t unit = depth >= 8 ? depth / 8 : 1; + size_t row_bytes = ((x2 - x1) * depth + 7) / 8; + size_t units = row_bytes / unit; + size_t worst_row = row_bytes + units / 128 + 1; + uint16_t at_once = _rows_at_once(row_bytes); + + uint8_t *header = message + WS_HEADER_SIZE; + uint8_t *data = header + RECT_HEADER_SIZE; + size_t room = PAYLOAD_SIZE - RECT_HEADER_SIZE; + size_t encoded = 0; + uint16_t rows = 0; + // Rendered rows can take a while, and a message of rows that encode small + // holds many of them. + while (y + rows < height && _row_dirty(y + rows) && encoded + worst_row <= room && + (rows == 0 || supervisor_ticks_ms64() < end_ms)) { + // The dirty rows that are sure to fit, cleared before they are read so a + // write after that marks them again. + uint16_t count = 0; + while (count < at_once && y + rows + count < height && _row_dirty(y + rows + count) && + encoded + (count + 1) * worst_row <= room) { + uint16_t row = y + rows + count; + dirty_rows[row / 8] &= ~(1 << (row % 8)); + count++; + } + size_t stride; + const uint8_t *src = _read_rows(y + rows, count, x1, x2, &stride); + for (uint16_t i = 0; i < count; i++) { + encoded += _encode_row(src + i * stride, units, unit, data + encoded); + } + rows += count; + } + uint8_t encoding = ENCODING_RLE; + if (encoded > rows * row_bytes) { + // Noisy pixels: send them as they are. + encoding = ENCODING_RAW; + for (uint16_t done = 0; done < rows;) { + uint16_t count = MIN(at_once, rows - done); + size_t stride; + const uint8_t *src = _read_rows(y + done, count, x1, x2, &stride); + for (uint16_t i = 0; i < count; i++) { + memcpy(data + (done + i) * row_bytes, src + i * stride, row_bytes); + } + done += count; + } + encoded = rows * row_bytes; + } + next_row = y + rows < height ? y + rows : 0; + + uint16_t w = x2 - x1; + header[0] = 1; + header[1] = encoding; + header[2] = x1 & 0xff; + header[3] = x1 >> 8; + header[4] = y & 0xff; + header[5] = y >> 8; + header[6] = w & 0xff; + header[7] = w >> 8; + header[8] = rows & 0xff; + header[9] = rows >> 8; + _finish_message(0x2, RECT_HEADER_SIZE + encoded); + return true; +} + +static void _read_commands(void) { + uint8_t c; + while (websocket_read_payload_byte(&ws, &c)) { + if (c == 'f' && target != NULL) { + _mark_all_dirty(); + } + } +} + +void display_stream_background(void) { + if (!streaming) { + return; + } + // A CLOSE from the client closes the socket without asking the co-processor. + if (common_hal_socketpool_socket_get_closed(&ws.socket)) { + _stop(); + return; + } + uint64_t start = supervisor_ticks_ms64(); + if (start - last_poll_ms >= POLL_MS) { + last_poll_ms = start; + if (!common_hal_socketpool_socket_get_connected(&ws.socket)) { + _stop(); + return; + } + } + // Some ports wake the web workflow until a socket's data is read, so read + // what the client sent every time. Control frames are answered with sends + // of their own, so not in the middle of a message. + if (message_start >= message_end) { + _read_commands(); + if (!streaming || common_hal_socketpool_socket_get_closed(&ws.socket)) { + _stop(); + return; + } + } + if (start - last_pass_ms < PASS_INTERVAL_MS) { + // Too soon after the last one, whatever woke the web workflow. + _pass_later(); + return; + } + last_pass_ms = start; + do { + if (message_start < message_end) { + int sent = socketpool_socket_send(&ws.socket, message + message_start, message_end - message_start); + if (sent == -MP_EAGAIN) { + break; + } + if (sent < 0) { + _stop(); + return; + } + message_start += sent; + continue; + } + _check_display(); + if (hello_needed) { + _build_hello(); + } else if (target == NULL || !_build_rect(start + PASS_BUDGET_MS)) { + break; + } + } while (supervisor_ticks_ms64() - start < PASS_BUDGET_MS); + // Come back for what's left: the rest of a message the socket couldn't take + // yet, or rows the budget ran out on. + if (message_start < message_end || (target != NULL && dirty_x1 != UINT16_MAX)) { + _pass_later(); + } else { + _no_pass_needed(); + } +} + +#endif diff --git a/supervisor/shared/web_workflow/display_stream.h b/supervisor/shared/web_workflow/display_stream.h new file mode 100644 index 00000000000..277d3be7e95 --- /dev/null +++ b/supervisor/shared/web_workflow/display_stream.h @@ -0,0 +1,49 @@ +// This file is part of the CircuitPython project: https://circuitpython.org +// +// SPDX-FileCopyrightText: Copyright (c) 2026 Tim Cocks for Adafruit Industries +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include "py/mpconfig.h" + +#if CIRCUITPY_WEB_WORKFLOW && CIRCUITPY_DISPLAYIO + +#include "shared-bindings/socketpool/Socket.h" +#include "shared-module/displayio/area.h" +#include "shared-module/displayio/display_core.h" + +// Streams the first display to one websocket client at /cp/display/. Framebuffer +// displays are read from their framebuffer. Bus displays have none, so changed +// rows are rendered again from the display's group. +// +// The client first gets a text message describing the display: +// {"width": 320, "height": 240, "format": "rgb565le", "lsb_first": true} +// format is one of rgb565le, rgb565be, rgb332, bgrx8888, rgbd4, gray1, gray2, +// gray4, gray8, or none when there is no display to stream. lsb_first says where +// the leftmost pixel of a byte is when pixels are smaller than a byte. A new one +// is sent when the display changes. +// +// Then each binary message is a rectangle of the display in its own format: +// uint8 type (1), uint8 encoding (0 raw, 1 run length), uint16 x, y, w, h +// (little endian), then the pixels row by row. Rows smaller than a byte are +// padded to whole bytes. With run length encoding the data is a series of +// control bytes in units of one pixel (one byte when pixels are smaller): +// c < 128 is followed by c + 1 units to copy, c >= 128 by one unit to repeat +// c - 126 times. +// +// The client may send the text "f" to have the whole display sent again. +// There is one client at a time: a new one takes over, and the old one is closed +// with code 4000 (WEBSOCKET_CLOSE_REPLACED). + +void display_stream_init(void); +// Takes the socket of a websocket upgrade request for /cp/display/. +void display_stream_handoff(socketpool_socket_obj_t *socket); +void display_stream_background(void); +// Called every tick while the stream has the tick enabled. +void display_stream_tick(void); +// Called by a display after it refreshed area. +void display_stream_area_updated(displayio_display_core_t *core, const displayio_area_t *area); + +#endif diff --git a/supervisor/shared/web_workflow/static/display.html b/supervisor/shared/web_workflow/static/display.html new file mode 100644 index 00000000000..47fabbe3b70 --- /dev/null +++ b/supervisor/shared/web_workflow/static/display.html @@ -0,0 +1,43 @@ + + + + Display + + + + + + +

 Display

+
+
+
+ +
+

+ + + + Connecting +

+
+
+
+ 

+      

+      

+        

+          Ctrl
+          
+          
+          
+        

+        
+        
+      

+    

+  

+  
+  
+
+
diff --git a/supervisor/shared/web_workflow/static/display.js b/supervisor/shared/web_workflow/static/display.js
new file mode 100644
index 00000000000..7063512e9f8
--- /dev/null
+++ b/supervisor/shared/web_workflow/static/display.js
@@ -0,0 +1,228 @@
+// Shows the device's display. The protocol is described in
+// supervisor/shared/web_workflow/display_stream.h.
+// Wrapped so its names don't clash with serial.js on the same page.
+(function() {
+  var canvas = document.getElementById("screen");
+  var context = canvas.getContext("2d");
+  var view = document.getElementById("view");
+  var status_text = document.getElementById("status");
+  var ws;
+  // Set when another page took the display over, so this one doesn't take it back.
+  var replaced = false;
+
+  var DEPTHS = {
+    "bgrx8888": 32, "rgb565le": 16, "rgb565be": 16, "rgb332": 8, "gray8": 8,
+    "rgbd4": 4, "gray4": 4, "gray2": 2, "gray1": 1
+  };
+
+  var info = null;
+  var depth = 0;
+  var unit = 1;
+  var image = null;
+  var pixels = null;
+  // Framebuffer value to canvas pixel (ABGR, which is RGBA in little endian memory).
+  var colors = null;
+  var updates = 0;
+  var received = 0;
+  // Degrees clockwise the picture is turned. Only the page turns it, the device
+  // sends the display as it is. Remembered by the browser where it can.
+  var rotation = 0;
+  try {
+    rotation = (parseInt(localStorage.getItem("display_rotation")) || 0) % 360;
+  } catch (e) {
+  }
+
+  // Fits the turned picture to the width of its column and three quarters of
+  // the window's height.
+  function layout() {
+    var turned = rotation % 180 != 0;
+    var shown_width = turned ? canvas.height : canvas.width;
+    var shown_height = turned ? canvas.width : canvas.height;
+    var scale = Math.min(view.clientWidth / shown_width, window.innerHeight * 0.75 / shown_height);
+    view.style.height = shown_height * scale + "px";
+    canvas.style.width = canvas.width * scale + "px";
+    canvas.style.height = canvas.height * scale + "px";
+    canvas.style.left = (view.clientWidth - canvas.width * scale) / 2 + "px";
+    canvas.style.top = (shown_height - canvas.height) * scale / 2 + "px";
+    canvas.style.transform = "rotate(" + rotation + "deg)";
+  }
+
+  function rgba(r, g, b) {
+    return (0xff000000 | (b << 16) | (g << 8) | r) >>> 0;
+  }
+
+  function setup(new_info) {
+    info = new_info;
+    depth = DEPTHS[info.format] || 0;
+    if (depth == 0) {
+      image = null;
+      status_text.textContent = "No display";
+      return;
+    }
+    unit = depth >= 8 ? depth / 8 : 1;
+    canvas.width = info.width;
+    canvas.height = info.height;
+    layout();
+    image = context.createImageData(info.width, info.height);
+    pixels = new Uint32Array(image.data.buffer);
+    colors = null;
+    if (info.format.startsWith("rgb565")) {
+      colors = new Uint32Array(65536);
+      for (var v = 0; v < 65536; v++) {
+        var r = v >> 11, g = (v >> 5) & 0x3f, b = v & 0x1f;
+        colors[v] = rgba((r << 3) | (r >> 2), (g << 2) | (g >> 4), (b << 3) | (b >> 2));
+      }
+    } else if (info.format == "rgb332") {
+      colors = new Uint32Array(256);
+      for (var v = 0; v < 256; v++) {
+        var r = v >> 5, g = (v >> 2) & 7, b = v & 3;
+        colors[v] = rgba(r * 255 / 7, g * 255 / 7, b * 255 / 3);
+      }
+    } else if (info.format == "rgbd4") {
+      colors = new Uint32Array(16);
+      for (var v = 0; v < 16; v++) {
+        colors[v] = rgba((v >> 3) & 1 ? 255 : 0, (v >> 2) & 1 ? 255 : 0, (v >> 1) & 1 ? 255 : 0);
+      }
+    } else if (info.format.startsWith("gray")) {
+      var levels = 1 << depth;
+      colors = new Uint32Array(levels);
+      for (var v = 0; v < levels; v++) {
+        var l = Math.round(v * 255 / (levels - 1));
+        colors[v] = rgba(l, l, l);
+      }
+    }
+  }
+
+  // Undoes the run length encoding into out.
+  function decode(data, offset, out) {
+    var o = 0;
+    while (offset < data.length && o < out.length) {
+      var c = data[offset++];
+      if (c < 128) {
+        var n = (c + 1) * unit;
+        out.set(data.subarray(offset, offset + n), o);
+        offset += n;
+        o += n;
+      } else {
+        var count = c - 126;
+        for (var i = 0; i < count; i++) {
+          for (var j = 0; j < unit; j++) {
+            out[o++] = data[offset + j];
+          }
+        }
+        offset += unit;
+      }
+    }
+  }
+
+  function draw(data) {
+    if (image == null || data[0] != 1) {
+      return;
+    }
+    var x = data[2] | (data[3] << 8);
+    var y = data[4] | (data[5] << 8);
+    var w = data[6] | (data[7] << 8);
+    var h = data[8] | (data[9] << 8);
+    var row_bytes = Math.ceil(w * depth / 8);
+    var rows;
+    if (data[1] == 1) {
+      rows = new Uint8Array(row_bytes * h);
+      decode(data, 10, rows);
+    } else {
+      rows = data.subarray(10);
+    }
+    var per_byte = 8 / depth;
+    var mask = (1 << depth) - 1;
+    for (var j = 0; j < h; j++) {
+      var src = j * row_bytes;
+      var dst = (y + j) * info.width + x;
+      for (var i = 0; i < w; i++) {
+        var p;
+        if (depth == 32) {
+          var s = src + i * 4;
+          p = rgba(rows[s + 2], rows[s + 1], rows[s]);
+        } else if (depth == 16) {
+          var s = src + i * 2;
+          var v = info.format == "rgb565le" ? rows[s] | (rows[s + 1] << 8) : (rows[s] << 8) | rows[s + 1];
+          p = colors[v];
+        } else if (depth == 8) {
+          p = colors[rows[src + i]];
+        } else {
+          var byte = rows[src + Math.floor(i / per_byte)];
+          var index = i % per_byte;
+          var shift = info.lsb_first ? index * depth : 8 - depth - index * depth;
+          p = colors[(byte >> shift) & mask];
+        }
+        pixels[dst + i] = p;
+      }
+    }
+    context.putImageData(image, 0, 0, x, y, w, h);
+    updates++;
+  }
+
+  function connect() {
+    status_text.textContent = "Connecting";
+    replaced = false;
+    ws = new WebSocket("ws://" + window.location.host + "/cp/display/");
+    ws.binaryType = "arraybuffer";
+    ws.onmessage = function(e) {
+      if (typeof e.data == "string") {
+        setup(JSON.parse(e.data));
+        return;
+      }
+      received += e.data.byteLength;
+      draw(new Uint8Array(e.data));
+    };
+    ws.onclose = function(e) {
+      if (e.code == 4000) {
+        replaced = true;
+        status_text.textContent = "Another page took over the display. Refresh to show it here.";
+        return;
+      }
+      status_text.textContent = "Disconnected";
+      setTimeout(connect, 2000);
+    };
+  }
+
+  document.getElementById("refresh").onclick = function() {
+    if (ws && ws.readyState == WebSocket.OPEN) {
+      ws.send("f");
+    } else if (replaced) {
+      connect();
+    }
+  };
+
+  function rotate(degrees) {
+    rotation = degrees % 360;
+    try {
+      if (rotation == 0) {
+        localStorage.removeItem("display_rotation");
+      } else {
+        localStorage.setItem("display_rotation", rotation);
+      }
+    } catch (e) {
+    }
+    layout();
+  }
+
+  document.getElementById("rotate").onclick = function() {
+    rotate(rotation + 90);
+  };
+  // Back to the display as the device sends it.
+  document.getElementById("unrotate").onclick = function() {
+    rotate(0);
+  };
+  window.addEventListener("resize", layout);
+  layout();
+
+  setInterval(function() {
+    if (ws && ws.readyState == WebSocket.OPEN && image != null) {
+      status_text.textContent = info.width + "x" + info.height + " " + info.format + ", " +
+        updates + " updates/s, " + (received / 1024).toFixed(1) + " KB/s";
+    }
+    updates = 0;
+    received = 0;
+  }, 1000);
+
+  connect();
+})();
diff --git a/supervisor/shared/web_workflow/static/serial.js b/supervisor/shared/web_workflow/static/serial.js
index c8ee2537bae..90f88d11fdb 100644
--- a/supervisor/shared/web_workflow/static/serial.js
+++ b/supervisor/shared/web_workflow/static/serial.js
@@ -7,7 +7,7 @@ var log = document.getElementById("log");
 
 function set_enabled(enabled) {
   input.disabled = !enabled;
-  var buttons = document.querySelectorAll("button");
+  var buttons = document.querySelectorAll("#controls button");
   for (button of buttons) {
     button.disabled = !enabled;
   }
@@ -15,6 +15,12 @@ function set_enabled(enabled) {
 
 set_enabled(false);
 
+// Scrolls the log's box rather than the page, which may hold more than the log.
+function scroll_to_end() {
+  var box = log.parentElement;
+  box.scrollTop = box.scrollHeight;
+}
+
 function onSubmit() {
     ws.send("\r");
     input.value = "";
@@ -48,11 +54,15 @@ ws.onmessage = function(e) {
   } else {
     log.textContent += e.data;
   }
-  document.querySelector("span").scrollIntoView();
+  scroll_to_end();
 };
 
-ws.onclose = function() {
+ws.onclose = function(e) {
   set_enabled(false);
+  if (e.code == 4000) {
+    log.textContent += "\n[Another page took over the serial connection. Reload to use it here.]\n";
+    scroll_to_end();
+  }
 };
 
 ws.onerror = function(e) {
diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c
index d445c115536..bae60054834 100644
--- a/supervisor/shared/web_workflow/web_workflow.c
+++ b/supervisor/shared/web_workflow/web_workflow.c
@@ -24,6 +24,7 @@
 #include "supervisor/shared/reload.h"
 #include "supervisor/shared/web_workflow/web_workflow.h"
 #include "supervisor/shared/web_workflow/websocket.h"
+#include "supervisor/shared/web_workflow/display_stream.h"
 #include "supervisor/shared/workflow.h"
 
 #include "shared-bindings/hashlib/__init__.h"
@@ -426,6 +427,9 @@ bool supervisor_start_web_workflow(void) {
         socketpool_socket_reset(&active);
 
         websocket_init();
+        #if CIRCUITPY_DISPLAYIO
+        display_stream_init();
+        #endif
     }
 
     initialized = pool.base.type == &socketpool_socketpool_type;
@@ -1193,6 +1197,10 @@ STATIC_FILE(edit_js);
 STATIC_FILE(style_css);
 STATIC_FILE(serial_html);
 STATIC_FILE(serial_js);
+#if CIRCUITPY_DISPLAYIO
+STATIC_FILE(display_html);
+STATIC_FILE(display_js);
+#endif
 STATIC_FILE(blinka_32x32_ico);
 
 static void _reply_static(socketpool_socket_obj_t *socket, _request *request, const uint8_t *response, size_t response_len, const char *content_type) {
@@ -1234,6 +1242,12 @@ static void _reply_websocket_upgrade(socketpool_socket_obj_t *socket, _request *
         "Connection: Upgrade\r\n",
         "Sec-WebSocket-Accept: ", encoded_accept, "\r\n",
         "\r\n", NULL);
+    #if CIRCUITPY_DISPLAYIO
+    if (strcmp(request->path, "/cp/display/") == 0) {
+        display_stream_handoff(socket);
+        return;
+    }
+    #endif
     websocket_handoff(socket);
     // socket is now closed and "disconnected".
 }
@@ -1470,6 +1484,20 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
             } else {
                 _REPLY_STATIC(socket, request, serial_html);
             }
+        #if CIRCUITPY_DISPLAYIO
+        } else if (strcmp(path, "/display/") == 0) {
+            if (!request->authenticated) {
+                if (_api_password[0] != '\0') {
+                    _reply_unauthorized(socket, request);
+                } else {
+                    _reply_forbidden(socket, request);
+                }
+            } else if (request->websocket) {
+                _reply_websocket_upgrade(socket, request);
+            } else {
+                _REPLY_STATIC(socket, request, display_html);
+            }
+        #endif
         } else {
             _reply_missing(socket, request);
         }
@@ -1486,6 +1514,10 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) {
             _REPLY_STATIC(socket, request, welcome_js);
         } else if (strcmp(request->path, "/serial.js") == 0) {
             _REPLY_STATIC(socket, request, serial_js);
+        #if CIRCUITPY_DISPLAYIO
+        } else if (strcmp(request->path, "/display.js") == 0) {
+            _REPLY_STATIC(socket, request, display_js);
+        #endif
         } else if (strcmp(request->path, "/edit.js") == 0) {
             _REPLY_STATIC(socket, request, edit_js);
         } else if (strcmp(request->path, "/style.css") == 0) {
@@ -1733,6 +1765,9 @@ void supervisor_web_workflow_background(void *data) {
 
     // Let the websocket code run.
     websocket_background();
+    #if CIRCUITPY_DISPLAYIO
+    display_stream_background();
+    #endif
 
     // Resume polling
     socketpool_socket_poll_resume();
diff --git a/supervisor/shared/web_workflow/websocket.c b/supervisor/shared/web_workflow/websocket.c
index 67b003b18a4..846f03ffb90 100644
--- a/supervisor/shared/web_workflow/websocket.c
+++ b/supervisor/shared/web_workflow/websocket.c
@@ -16,17 +16,6 @@
 #include "supervisor/shared/status_bar.h"
 #endif
 
-typedef struct {
-    socketpool_socket_obj_t socket;
-    uint8_t opcode;
-    uint8_t frame_len;
-    uint8_t payload_len_size;
-    bool masked;
-    uint8_t mask[4];
-    int frame_index;
-    size_t payload_remaining;
-} _websocket;
-
 // Buffer the incoming serial data in the background so that we can look for the
 // interrupt character.
 static ringbuf_t _incoming_ringbuf;
@@ -34,7 +23,7 @@ static uint8_t _buf[16];
 // make sure background is not called recursively
 static bool in_web_background = false;
 
-static _websocket cp_serial;
+static websocket_t cp_serial;
 
 void websocket_init(void) {
     socketpool_socket_reset(&cp_serial.socket);
@@ -42,15 +31,33 @@ void websocket_init(void) {
     ringbuf_init(&_incoming_ringbuf, _buf, sizeof(_buf));
 }
 
-void websocket_handoff(socketpool_socket_obj_t *socket) {
-    if (!common_hal_socketpool_socket_get_closed(&cp_serial.socket)) {
-        common_hal_socketpool_socket_close(&cp_serial.socket);
+void websocket_send_close(websocket_t *ws, uint16_t code) {
+    if (common_hal_socketpool_socket_get_closed(&ws->socket) ||
+        !common_hal_socketpool_socket_get_connected(&ws->socket)) {
+        return;
     }
+    const uint8_t frame[4] = { 1 << 7 | 0x8, 2, code >> 8, code & 0xff };
+    // Only what the socket takes now: a client that stopped reading must not
+    // hold everything up.
+    socketpool_socket_send(&ws->socket, frame, sizeof(frame));
+}
 
-    socketpool_socket_move(socket, &cp_serial.socket);
-    cp_serial.opcode = 0;
-    cp_serial.frame_index = 0;
-    cp_serial.frame_len = 2;
+void websocket_take_socket(websocket_t *ws, socketpool_socket_obj_t *socket) {
+    if (!common_hal_socketpool_socket_get_closed(&ws->socket)) {
+        // Tell the page it was replaced, so it doesn't reconnect and take the
+        // connection back.
+        websocket_send_close(ws, WEBSOCKET_CLOSE_REPLACED);
+        common_hal_socketpool_socket_close(&ws->socket);
+    }
+
+    socketpool_socket_move(socket, &ws->socket);
+    ws->opcode = 0;
+    ws->frame_index = 0;
+    ws->frame_len = 2;
+}
+
+void websocket_handoff(socketpool_socket_obj_t *socket) {
+    websocket_take_socket(&cp_serial, socket);
 
     #if CIRCUITPY_STATUS_BAR
     // Send the title bar for the new client.
@@ -64,95 +71,95 @@ bool websocket_connected(void) {
            common_hal_socketpool_socket_get_connected(&cp_serial.socket);
 }
 
-static bool _read_byte(uint8_t *c) {
-    int len = socketpool_socket_recv_into(&cp_serial.socket, c, 1);
+static bool _read_byte(websocket_t *ws, uint8_t *c) {
+    int len = socketpool_socket_recv_into(&ws->socket, c, 1);
     if (len < 1) {
         return false;
     }
     return true;
 }
 
-static void _read_next_frame_header(void) {
-    uint8_t h;
-    if (cp_serial.frame_index == 0 && _read_byte(&h)) {
-        cp_serial.frame_index++;
-        cp_serial.opcode = h & 0xf;
+static void _read_next_frame_header(websocket_t *ws) {
+    uint8_t h = 0;
+    if (ws->frame_index == 0 && _read_byte(ws, &h)) {
+        ws->frame_index++;
+        ws->opcode = h & 0xf;
     }
-    if (cp_serial.frame_index == 1 && _read_byte(&h)) {
-        cp_serial.frame_index++;
+    if (ws->frame_index == 1 && _read_byte(ws, &h)) {
+        ws->frame_index++;
         uint8_t len = h & 0x7f;
-        cp_serial.masked = (h >> 7) == 1;
+        ws->masked = (h >> 7) == 1;
         if (len <= 125) {
-            cp_serial.payload_remaining = len;
-            cp_serial.payload_len_size = 0;
+            ws->payload_remaining = len;
+            ws->payload_len_size = 0;
         } else if (len == 126) { // 16 bit length
-            cp_serial.payload_len_size = 2;
+            ws->payload_len_size = 2;
         } else if (len == 127) { // 64 bit length
-            cp_serial.payload_len_size = 8;
+            ws->payload_len_size = 8;
         }
-        cp_serial.frame_len = 2 + cp_serial.payload_len_size;
-        if (cp_serial.masked) {
-            cp_serial.frame_len += 4;
+        ws->frame_len = 2 + ws->payload_len_size;
+        if (ws->masked) {
+            ws->frame_len += 4;
         }
     }
-    while (cp_serial.frame_index >= 2 &&
-           cp_serial.frame_index < (cp_serial.payload_len_size + 2) &&
-           _read_byte(&h)) {
-        cp_serial.frame_index++;
-        cp_serial.payload_remaining = cp_serial.payload_remaining << 8 | h;
+    while (ws->frame_index >= 2 &&
+           ws->frame_index < (ws->payload_len_size + 2) &&
+           _read_byte(ws, &h)) {
+        ws->frame_index++;
+        ws->payload_remaining = ws->payload_remaining << 8 | h;
     }
-    int mask_start = cp_serial.payload_len_size + 2;
-    while (cp_serial.frame_index >= mask_start &&
-           cp_serial.frame_index < cp_serial.frame_len &&
-           _read_byte(&h)) {
-        size_t mask_offset = cp_serial.frame_index - mask_start;
-        cp_serial.mask[mask_offset] = h;
-        cp_serial.frame_index++;
+    int mask_start = ws->payload_len_size + 2;
+    while (ws->frame_index >= mask_start &&
+           ws->frame_index < ws->frame_len &&
+           _read_byte(ws, &h)) {
+        size_t mask_offset = ws->frame_index - mask_start;
+        ws->mask[mask_offset] = h;
+        ws->frame_index++;
     }
     // Reply to PINGs and CLOSE.
-    while ((cp_serial.opcode == 0x8 ||
-            cp_serial.opcode == 0x9) &&
-           cp_serial.frame_index >= cp_serial.frame_len) {
+    while ((ws->opcode == 0x8 ||
+            ws->opcode == 0x9) &&
+           ws->frame_index >= ws->frame_len) {
 
-        if (cp_serial.frame_index == cp_serial.frame_len) {
+        if (ws->frame_index == ws->frame_len) {
             uint8_t opcode = 0x8; // CLOSE
-            if (cp_serial.opcode == 0x9) {
+            if (ws->opcode == 0x9) {
                 opcode = 0xA; // PONG
             }
             uint8_t frame_header[2];
             frame_header[0] = 1 << 7 | opcode;
-            frame_header[1] = cp_serial.payload_remaining;
-            web_workflow_send_raw(&cp_serial.socket, cp_serial.opcode != 0x9, (const uint8_t *)frame_header, 2);
+            frame_header[1] = ws->payload_remaining;
+            web_workflow_send_raw(&ws->socket, ws->opcode != 0x9, (const uint8_t *)frame_header, 2);
         }
 
-        if (cp_serial.payload_remaining > 0 && _read_byte(&h)) {
+        if (ws->payload_remaining > 0 && _read_byte(ws, &h)) {
             // Send the payload back to the client.
-            cp_serial.frame_index++;
-            cp_serial.payload_remaining--;
-            web_workflow_send_raw(&cp_serial.socket, false, &h, 1);
+            ws->frame_index++;
+            ws->payload_remaining--;
+            web_workflow_send_raw(&ws->socket, false, &h, 1);
         }
 
-        if (cp_serial.payload_remaining == 0) {
-            cp_serial.frame_index = 0;
-            if (cp_serial.opcode == 0x8) {
-                common_hal_socketpool_socket_close(&cp_serial.socket);
+        if (ws->payload_remaining == 0) {
+            ws->frame_index = 0;
+            if (ws->opcode == 0x8) {
+                common_hal_socketpool_socket_close(&ws->socket);
             }
         }
     }
 }
 
-static bool _read_next_payload_byte(uint8_t *c) {
-    _read_next_frame_header();
-    if (cp_serial.opcode == 0x1 &&
-        cp_serial.frame_index >= cp_serial.frame_len &&
-        cp_serial.payload_remaining > 0) {
-        if (_read_byte(c)) {
-            uint8_t mask_offset = (cp_serial.frame_index - cp_serial.frame_len) % 4;
-            *c ^= cp_serial.mask[mask_offset];
-            cp_serial.frame_index++;
-            cp_serial.payload_remaining--;
-            if (cp_serial.payload_remaining == 0) {
-                cp_serial.frame_index = 0;
+bool websocket_read_payload_byte(websocket_t *ws, uint8_t *c) {
+    _read_next_frame_header(ws);
+    if (ws->opcode == 0x1 &&
+        ws->frame_index >= ws->frame_len &&
+        ws->payload_remaining > 0) {
+        if (_read_byte(ws, c)) {
+            uint8_t mask_offset = (ws->frame_index - ws->frame_len) % 4;
+            *c ^= ws->mask[mask_offset];
+            ws->frame_index++;
+            ws->payload_remaining--;
+            if (ws->payload_remaining == 0) {
+                ws->frame_index = 0;
             }
             return true;
         }
@@ -176,7 +183,7 @@ char websocket_read_char(void) {
     return -1;
 }
 
-static void _websocket_send(_websocket *ws, const char *text, size_t len) {
+static void _websocket_send(websocket_t *ws, const char *text, size_t len) {
     if (!websocket_connected()) {
         return;
     }
@@ -225,7 +232,7 @@ void websocket_background(void) {
     in_web_background = true;
     uint8_t c;
     while (ringbuf_num_empty(&_incoming_ringbuf) > 0 &&
-           _read_next_payload_byte(&c)) {
+           websocket_read_payload_byte(&cp_serial, &c)) {
         if (c == mp_interrupt_char) {
             ringbuf_clear(&_incoming_ringbuf);
             mp_sched_keyboard_interrupt();
diff --git a/supervisor/shared/web_workflow/websocket.h b/supervisor/shared/web_workflow/websocket.h
index bbf30a4611d..e31e20c5f34 100644
--- a/supervisor/shared/web_workflow/websocket.h
+++ b/supervisor/shared/web_workflow/websocket.h
@@ -11,6 +11,30 @@
 
 #include "shared-bindings/socketpool/Socket.h"
 
+typedef struct {
+    socketpool_socket_obj_t socket;
+    uint8_t opcode;
+    uint8_t frame_len;
+    uint8_t payload_len_size;
+    bool masked;
+    uint8_t mask[4];
+    int frame_index;
+    size_t payload_remaining;
+} websocket_t;
+
+// Close code sent to a client when a newer one takes over its connection.
+#define WEBSOCKET_CLOSE_REPLACED (4000)
+
+// Sends a CLOSE frame with code if the socket can take it without waiting. The
+// caller closes the socket.
+void websocket_send_close(websocket_t *ws, uint16_t code);
+// Moves an upgraded socket into ws, closing the one it had. A connected client
+// is sent WEBSOCKET_CLOSE_REPLACED first.
+void websocket_take_socket(websocket_t *ws, socketpool_socket_obj_t *socket);
+// Reads the next byte of a text message. PINGs are answered and a CLOSE closes
+// the socket. Returns false when no byte is available yet.
+bool websocket_read_payload_byte(websocket_t *ws, uint8_t *c);
+
 void websocket_init(void);
 void websocket_handoff(socketpool_socket_obj_t *socket);
 bool websocket_connected(void);
diff --git a/supervisor/supervisor.mk b/supervisor/supervisor.mk
index 2d881dfe070..bb51b033666 100644
--- a/supervisor/supervisor.mk
+++ b/supervisor/supervisor.mk
@@ -234,7 +234,8 @@ $(BUILD)/autogen_web_workflow_static.c: ../../tools/gen_web_workflow_static.py $
 
 ifeq ($(CIRCUITPY_WEB_WORKFLOW),1)
   SRC_SUPERVISOR += supervisor/shared/web_workflow/web_workflow.c \
-                    supervisor/shared/web_workflow/websocket.c
+                    supervisor/shared/web_workflow/websocket.c \
+                    supervisor/shared/web_workflow/display_stream.c
   SRC_SUPERVISOR += $(BUILD)/autogen_web_workflow_static.c
 endif