macOS guests 5/6: ingress and fixed addresses for macOS guests

Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded
+.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy build-caddy-darwin ensure-caddy-binaries release-prep clean build-embedded
 
 # Directory where local binaries will be installed
 BIN_DIR ?= $(CURDIR)/bin
@@ -93,6 +93,13 @@ build-caddy-binaries: $(XCADDY)
 	@chmod +x lib/ingress/binaries/caddy/$(CADDY_VERSION)/*/caddy
 	@echo "Caddy binaries built successfully with DNS modules"
 
+# Build Caddy with the same DNS modules for macOS hosts, which find it next to
+# hypeman (bin/caddy) or on PATH instead of embedding it.
+build-caddy-darwin: $(XCADDY) | $(BIN_DIR)
+	GOOS=darwin GOARCH=arm64 $(XCADDY) build $(CADDY_VERSION) \
+		$(CADDY_DNS_MODULES) \
+		--output $(BIN_DIR)/caddy
+
 # Build Caddy for current architecture only (faster for development)
 build-caddy: $(XCADDY)
 	@echo "Building Caddy $(CADDY_VERSION) with DNS modules for current architecture..."

cmd/api/api/capabilities_rosetta_darwin_arm64.go

@@ -2,7 +2,7 @@
 
 package api
 
-import "github.com/Code-Hex/vz/v3"
+import "github.com/Code-Hex/vz/v4"
 
 // rosettaInstalled reports whether Rosetta translation for Linux guests is
 // installed and usable right now, using the same Virtualization.framework

cmd/api/api/ingress.go

@@ -36,12 +36,6 @@ func (s *ApiService) ListIngresses(ctx context.Context, request oapi.ListIngress
 
 // CreateIngress creates a new ingress resource
 func (s *ApiService) CreateIngress(ctx context.Context, request oapi.CreateIngressRequestObject) (oapi.CreateIngressResponseObject, error) {
-	if s.Config != nil && s.Config.MacOSOnly {
-		return oapi.CreateIngress501JSONResponse{Code: "unsupported", Message: "ingress is not implemented in macOS-only mode"}, nil
-	}
-	if reason := s.macOSIngressTargetReason(ctx, request.Body.Rules); reason != "" {
-		return oapi.CreateIngress501JSONResponse{Code: "unsupported", Message: reason}, nil
-	}
 	log := logger.FromContext(ctx)
 
 	// Convert OAPI request to domain request

cmd/api/api/macos.go

@@ -10,8 +10,6 @@ import (
 	"runtime"
 
 	"github.com/kernel/hypeman/lib/images"
-	"github.com/kernel/hypeman/lib/ingress"
-	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/oapi"
 )
 
@@ -35,34 +33,6 @@ func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error
 	return img.MacOS != nil, nil
 }
 
-// macOSIngressTargetReason refuses routes to macOS guests. Their addresses are NAT
-// leases, and no ingress path has been proven for them, so mixed-mode servers must
-// not route public traffic to them. A target that cannot be looked up is refused.
-// Pattern hostnames resolve their target per request, where the ingress resolver
-// refuses macOS guests, and a missing target is left to the ingress manager's 400.
-func (s *ApiService) macOSIngressTargetReason(ctx context.Context, rules []oapi.IngressRule) string {
-	if !macOSHost || s.InstanceManager == nil {
-		return ""
-	}
-	for _, rule := range rules {
-		match := ingress.IngressMatch{Hostname: rule.Match.Hostname}
-		if match.IsPattern() {
-			continue
-		}
-		inst, err := s.InstanceManager.GetInstance(ctx, rule.Target.Instance)
-		if errors.Is(err, instances.ErrNotFound) {
-			continue
-		}
-		if err != nil {
-			return "cannot verify ingress target is not a macOS guest"
-		}
-		if inst.MacOS != nil {
-			return "ingress cannot target macOS guests"
-		}
-	}
-	return ""
-}
-
 // Linux defaults for CreateInstanceRequest in openapi.yaml.
 const (
 	linuxDefaultSize        = "1GB"

cmd/api/api/macos_test.go

@@ -2,8 +2,6 @@ package api
 
 import (
 	"context"
-	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -18,14 +16,11 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestMacOSOnlyRejectsBuildersAndIngress(t *testing.T) {
+func TestMacOSOnlyRejectsBuilders(t *testing.T) {
 	s := &ApiService{Config: &config.Config{MacOSOnly: true}}
 	build, err := s.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{})
 	require.NoError(t, err)
 	require.IsType(t, oapi.CreateBuild501JSONResponse{}, build)
-	ingress, err := s.CreateIngress(context.Background(), oapi.CreateIngressRequestObject{})
-	require.NoError(t, err)
-	require.IsType(t, oapi.CreateIngress501JSONResponse{}, ingress)
 }
 
 func TestMacOSExecRejectedBeforeWebsocketUpgrade(t *testing.T) {
@@ -177,37 +172,3 @@ type macOSImageManager struct{ images.Manager }
 func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) {
 	return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil
 }
-
-type macOSIngressTargetManager struct {
-	instances.Manager
-	inst *instances.Instance
-	err  error
-}
-
-func (m macOSIngressTargetManager) GetInstance(context.Context, string) (*instances.Instance, error) {
-	return m.inst, m.err
-}
-
-func TestIngressRejectsMacOSTargets(t *testing.T) {
-	useMacOSHost(t)
-	rule := oapi.IngressRule{Match: oapi.IngressMatch{Hostname: "app.example.test"}, Target: oapi.IngressTarget{Instance: "mac", Port: 8080}}
-	macos := &instances.Instance{StoredMetadata: instances.StoredMetadata{MacOS: &images.MacOSImage{}}}
-	s := &ApiService{InstanceManager: macOSIngressTargetManager{inst: macos}}
-	require.Contains(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "macOS")
-
-	linux := &instances.Instance{}
-	s = &ApiService{InstanceManager: macOSIngressTargetManager{inst: linux}}
-	require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}))
-
-	s = &ApiService{InstanceManager: macOSIngressTargetManager{err: errors.New("lookup failed")}}
-	require.NotEmpty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "an unverifiable target is refused")
-
-	s = &ApiService{InstanceManager: macOSIngressTargetManager{err: fmt.Errorf("lookup: %w", instances.ErrNotFound)}}
-	require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{rule}), "the ingress manager reports a missing target as 400")
-
-	pattern := rule
-	pattern.Match.Hostname = "{instance}.example.test"
-	pattern.Target.Instance = "{instance}"
-	s = &ApiService{InstanceManager: macOSIngressTargetManager{err: errors.New("lookup failed")}}
-	require.Empty(t, s.macOSIngressTargetReason(context.Background(), []oapi.IngressRule{pattern}), "pattern targets resolve per request, where the resolver refuses macOS guests")
-}

cmd/api/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -67,6 +68,8 @@ type NetworkConfig struct {
 	DNSServer               string `koanf:"dns_server"`
 	UploadBurstMultiplier   int    `koanf:"upload_burst_multiplier"`
 	DownloadBurstMultiplier int    `koanf:"download_burst_multiplier"`
+	// MacOSSubnetCIDR is the pool macOS guest networks come from: one /29 per guest.
+	MacOSSubnetCIDR string `koanf:"macos_subnet_cidr"`
 }
 
 // CaddyConfig holds Caddy reverse-proxy / ingress settings.
@@ -338,6 +341,7 @@ func defaultConfig() *Config {
 			DNSServer:               "1.1.1.1",
 			UploadBurstMultiplier:   4,
 			DownloadBurstMultiplier: 4,
+			MacOSSubnetCIDR:         "192.168.128.0/20",
 		},
 
 		Caddy: CaddyConfig{
@@ -594,6 +598,11 @@ func (c *Config) Validate() error {
 	if c.MacOSOnly && (runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" || c.Hypervisor.Default != "vz") {
 		return fmt.Errorf("macos_only requires vz on Apple silicon")
 	}
+	if pool, err := netip.ParsePrefix(c.Network.MacOSSubnetCIDR); err != nil || !pool.Addr().Is4() || pool.Bits() < 8 || pool.Bits() > 29 {
+		return fmt.Errorf("network.macos_subnet_cidr must be an IPv4 prefix from /8 to /29, got %q", c.Network.MacOSSubnetCIDR)
+	} else if linux, err := netip.ParsePrefix(c.Network.SubnetCIDR); err == nil && linux.Overlaps(pool) {
+		return fmt.Errorf("network.macos_subnet_cidr %s overlaps network.subnet_cidr %s", pool, linux)
+	}
 	if err := c.validateAPIListenPorts(); err != nil {
 		return err
 	}

cmd/api/config/config_test.go

@@ -605,3 +605,16 @@ func TestValidateBuildersConfig(t *testing.T) {
 		t.Fatalf("expected parsed idle ttl 24h, got %s", cfg.Builders.IdleTTLDuration)
 	}
 }
+
+func TestValidateMacOSSubnetCIDR(t *testing.T) {
+	if err := defaultConfig().Validate(); err != nil {
+		t.Fatalf("expected the default macOS guest pool to validate, got %v", err)
+	}
+	for _, pool := range []string{"", "nonsense", "fd00::/64", "192.168.128.0/30", "10.100.4.0/24"} {
+		cfg := defaultConfig()
+		cfg.Network.MacOSSubnetCIDR = pool
+		if err := cfg.Validate(); err == nil {
+			t.Fatalf("expected macOS guest pool %q to be rejected", pool)
+		}
+	}
+}

cmd/api/hypervisor_check_darwin.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"runtime"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 )
 
 // checkHypervisorAccess verifies Virtualization.framework is available on macOS

cmd/api/main.go

@@ -407,15 +407,11 @@ func run() error {
 
 	// Initialize ingress manager (starts Caddy daemon and DNS server for dynamic upstreams)
 	logger.Info("Initializing ingress manager...")
-	if !cfg.MacOSOnly {
-		if err := app.IngressManager.Initialize(app.Ctx); err != nil {
-			logger.Error("failed to initialize ingress manager", "error", err)
-			return fmt.Errorf("initialize ingress manager: %w", err)
-		}
-		logger.Info("Ingress manager initialized", "listen_addr", cfg.Caddy.ListenAddress, "admin", app.IngressManager.AdminURL())
-	} else {
-		logger.Info("macOS-only mode: ingress is unsupported; skipping Caddy and DNS startup")
+	if err := app.IngressManager.Initialize(app.Ctx); err != nil {
+		logger.Error("failed to initialize ingress manager", "error", err)
+		return fmt.Errorf("initialize ingress manager: %w", err)
 	}
+	logger.Info("Ingress manager initialized", "listen_addr", cfg.Caddy.ListenAddress, "admin", app.IngressManager.AdminURL())
 
 	// Create router
 	r := chi.NewRouter()

cmd/vz-shim/macos_arm64.go

@@ -6,7 +6,7 @@ import (
 	"encoding/base64"
 	"fmt"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 

cmd/vz-shim/macos_other.go

@@ -4,7 +4,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 

cmd/vz-shim/macos_vm.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 

cmd/vz-shim/main.go

@@ -18,7 +18,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 
@@ -131,23 +131,34 @@ func main() {
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
 
-	// Monitor VM state
-	stateChanged := vm.StateChangedNotify()
+	// Monitor VM state. Notifications can be dropped and their values can be stale,
+	// so each one, and the start, re-reads the current state.
+	stateChanged := make(chan vz.VirtualMachineState, 8)
+	stopNotify, err := vm.StateChangedNotify(stateChanged)
+	if err != nil {
+		slog.Error("failed to watch VM state", "error", err)
+		os.Exit(1)
+	}
+	defer stopNotify()
 	go func() {
 		for {
-			select {
-			case <-ctx.Done():
-				return
-			case newState := <-stateChanged:
-				slog.Info("VM state changed", "state", newState)
-				if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError {
-					if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() {
-						writeStopReceipt(config.StopReceiptPath)
-					}
-					slog.Info("VM stopped, shutting down shim")
-					cancel()
+			newState := vm.State()
+			if newState != vz.VirtualMachineStateStopped && newState != vz.VirtualMachineStateError {
+				select {
+				case <-ctx.Done():
 					return
+				case <-stateChanged:
 				}
+				newState = vm.State()
+				slog.Info("VM state changed", "state", newState)
+			}
+			if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError {
+				if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() {
+					writeStopReceipt(config.StopReceiptPath)
+				}
+				slog.Info("VM stopped, shutting down shim")
+				cancel()
+				return
 			}
 		}
 	}()

cmd/vz-shim/rosetta_arm64.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 

cmd/vz-shim/rosetta_arm64_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

cmd/vz-shim/rosetta_other.go

@@ -5,7 +5,7 @@ package main
 import (
 	"fmt"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )
 

cmd/vz-shim/save_restore_arm64.go

@@ -5,7 +5,7 @@ package main
 import (
 	"fmt"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 )
 
 func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error {

cmd/vz-shim/save_restore_unsupported.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"runtime"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 )
 
 func validateSaveRestoreSupport(vmConfig *vz.VirtualMachineConfiguration) error {

cmd/vz-shim/server.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/Code-Hex/vz/v3"
+	"github.com/Code-Hex/vz/v4"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
 )