From 4d74ad61136e4a3b5c526a49c1b3123251ab37af Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 12:40:19 +0200 Subject: [PATCH 1/2] Apply cooldown to Go modules With cooldown on, @v/list leaves out versions whose .info time is inside the window, and @latest falls back to the newest release out of cooldown when upstream's answer is too new. Each listed version's .info is read once and its time is stored in the versions table, as npm does, except for modules whose path has uppercase letters, since golang PURLs are lowercased. A .info, .mod or .zip for a version in cooldown, and a query such as a branch name that resolves to one, gets 403. The go command moves on to the next GOPROXY entry after 404 or 410, so a 404 would let GOPROXY=proxy,direct fetch the version directly. Go has no publish time: .info carries the commit time of the revision, so a version tagged on an old commit counts as old. The docs say so. Part of #132 --- README.md | 4 +- docs/configuration.md | 4 +- internal/handler/go.go | 27 ++- internal/handler/go_cooldown.go | 252 +++++++++++++++++++++++++++ internal/handler/go_cooldown_test.go | 191 ++++++++++++++++++++ 5 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 internal/handler/go_cooldown.go create mode 100644 internal/handler/go_cooldown_test.go diff --git a/README.md b/README.md index 0553e4a1..9f54b746 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | npm | JavaScript | Yes | ✓ | | Cargo | Rust | Yes | ✓ | | RubyGems | Ruby | Yes | ✓ | -| Go proxy | Go | | ✓ | +| Go proxy | Go | Yes† | ✓ | | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | @@ -77,6 +77,8 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in \* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. +† Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. + ## Install ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 4da6f634..eb99cd9e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -980,7 +980,9 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. + +Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/go.go b/internal/handler/go.go index c8ae81ae..85cf7e9f 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -84,17 +84,28 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { rest := path[idx+4:] // after "/@v/" decodedMod := decodeGoModule(module) + cooldown := h.cooldownEnabled() switch { + case rest == "list" && cooldown: + h.serveCooldownList(w, r, module, decodedMod) + case rest == "list": // GET /{module}/@v/list - list versions h.proxyCached(w, r, decodedMod+"/@v/list") + case strings.HasSuffix(rest, ".info") && cooldown: + h.serveCooldownInfo(w, r, module, decodedMod, rest) + case strings.HasSuffix(rest, ".info"): // GET /{module}/@v/{version}.info - version metadata h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file + if version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")); err == nil && h.versionInCooldown(r.Context(), module, decodedMod, version) { + h.refuseCooldown(w, decodedMod, version) + return + } h.proxyCached(w, r, decodedMod+"/@v/"+rest) case strings.HasSuffix(rest, ".zip"): @@ -111,6 +122,10 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { // Check for @latest if strings.HasSuffix(path, "/@latest") { module := strings.TrimSuffix(path, "/@latest") + if h.cooldownEnabled() { + h.serveCooldownLatest(w, r, module, decodeGoModule(module)) + return + } h.proxyCached(w, r, decodeGoModule(module)+"/@latest") return } @@ -130,9 +145,15 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for // v1.0.0-RC1), and the zip is cached under that form, so the denylist // is checked against the version itself. - if unescaped, err := gomodule.UnescapeVersion(version); err == nil && h.proxy.versionDenied("golang", decodedModule, unescaped) { - http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) - return + if unescaped, err := gomodule.UnescapeVersion(version); err == nil { + if h.proxy.versionDenied("golang", decodedModule, unescaped) { + http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) + return + } + if h.versionInCooldown(r.Context(), module, decodedModule, unescaped) { + h.refuseCooldown(w, decodedModule, unescaped) + return + } } downloadURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go new file mode 100644 index 00000000..f73a897f --- /dev/null +++ b/internal/handler/go_cooldown.go @@ -0,0 +1,252 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + gomodule "golang.org/x/mod/module" + "golang.org/x/mod/semver" + "golang.org/x/sync/errgroup" +) + +// Go cooldown measures a version by the Time in its .info, which the module +// proxy takes from the commit the version points at. That is when the code +// was committed, not when the version was tagged or first fetched. +// +// Versions in cooldown are left out of @v/list and @latest, and their .info, +// .mod and .zip are answered with 403. The go command falls back to the next +// GOPROXY entry on 404 and 410, so with GOPROXY=proxy,direct a 404 would let +// it fetch the version directly. + +// goInfoFetchLimit bounds the .info requests made at once to filter a list. +const goInfoFetchLimit = 8 + +const errGoCooldown = "version is in cooldown" + +// goInfo is the JSON served for a version's .info and for @latest. +type goInfo struct { + Version string `json:"Version"` + Time time.Time `json:"Time"` +} + +func (h *GoHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func (h *GoHandler) allowedAt(module string, published time.Time) bool { + if published.IsZero() { + return true + } + return h.proxy.Cooldown.IsAllowed("golang", canonicalPackagePURL("golang", module), published) +} + +// writeGoMetadataError answers a failed upstream metadata fetch. +func (h *GoHandler) writeGoMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("go metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) +} + +func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, rest string) ([]byte, error) { + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "golang", module+"/"+rest, h.upstreamURL+"/"+escapedModule+"/"+rest, "*/*") + return body, err +} + +// storesGoTimes reports whether publish times for module can be kept in the +// versions table. Its rows are keyed by PURL, which lowercases golang names, +// so modules that differ only in case would share them; such modules read +// their times from the metadata cache instead. +func (h *GoHandler) storesGoTimes(module string) bool { + return h.proxy.DB != nil && module == strings.ToLower(module) +} + +// publishTime returns a version's time from the versions table, or from its +// .info, which is then stored. A version whose time can't be read gets the +// zero time and is let through, as in the other ecosystems. +func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) time.Time { + versionPURL := canonicalVersionPURL("golang", module, version) + if h.storesGoTimes(module) { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ver.PublishedAt.Time + } + } + + escapedVersion, err := gomodule.EscapeVersion(version) + if err != nil { + return time.Time{} + } + body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/"+escapedVersion+".info") + if err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch go version info", + "module", module, "version", version, "error", err) + return time.Time{} + } + var info goInfo + if err := json.Unmarshal(body, &info); err != nil || info.Time.IsZero() { + return time.Time{} + } + h.storePublishTime(module, version, info.Time) + return info.Time +} + +func (h *GoHandler) storePublishTime(module, version string, published time.Time) { + if !h.storesGoTimes(module) { + return + } + if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("golang", module, version), canonicalPackagePURL("golang", module), published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store go publish time", + "module", module, "version", version, "error", err) + } +} + +// versionInCooldown reports whether an exact version is still in cooldown. +func (h *GoHandler) versionInCooldown(ctx context.Context, escapedModule, module, version string) bool { + if !h.cooldownEnabled() { + return false + } + return !h.allowedAt(module, h.publishTime(ctx, escapedModule, module, version)) +} + +func (h *GoHandler) refuseCooldown(w http.ResponseWriter, module, version string) { + h.proxy.Logger.Info("cooldown: withholding go module version", "module", module, "version", version) + http.Error(w, errGoCooldown+": "+canonicalVersionPURL("golang", module, version), http.StatusForbidden) +} + +// allowedVersions returns the listed versions of module that are out of +// cooldown, in their upstream order. +func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module string) ([]string, error) { + body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/list") + if err != nil { + return nil, err + } + var versions []string + for line := range strings.SplitSeq(string(body), "\n") { + if v := strings.TrimSpace(line); v != "" { + versions = append(versions, v) + } + } + + keep := make([]bool, len(versions)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(goInfoFetchLimit) + for i, v := range versions { + g.Go(func() error { + keep[i] = h.allowedAt(module, h.publishTime(gctx, escapedModule, module, v)) + return nil + }) + } + _ = g.Wait() + + allowed := versions[:0] + for i, v := range versions { + if keep[i] { + allowed = append(allowed, v) + } else { + h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) + } + } + return allowed, nil +} + +// serveCooldownList serves @v/list without versions in cooldown. +func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { + allowed, err := h.allowedVersions(r.Context(), escapedModule, module) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + w.Header().Set(headerContentType, "text/plain; charset=utf-8") + if len(allowed) > 0 { + _, _ = w.Write([]byte(strings.Join(allowed, "\n") + "\n")) + } +} + +// serveCooldownInfo serves a .info, refusing it when the version it names is +// in cooldown. rest is the request after /@v/: an escaped version or a query +// such as a branch name, followed by .info. +func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, escapedModule, module, rest string) { + body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@v/"+rest) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + var info goInfo + if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { + if !h.allowedAt(module, info.Time) { + h.refuseCooldown(w, module, info.Version) + return + } + if !info.Time.IsZero() && semver.IsValid(info.Version) { + h.storePublishTime(module, info.Version, info.Time) + } + } + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write(body) +} + +// serveCooldownLatest serves @latest. When upstream's answer is in cooldown, +// it serves the newest listed version out of cooldown instead: the highest +// release, or else the highest pre-release, which is how the go command +// picks from a list. +func (h *GoHandler) serveCooldownLatest(w http.ResponseWriter, r *http.Request, escapedModule, module string) { + body, err := h.fetchGoMetadata(r.Context(), escapedModule, module, "@latest") + if err != nil { + h.writeGoMetadataError(w, err) + return + } + var info goInfo + if err := json.Unmarshal(body, &info); err != nil || h.allowedAt(module, info.Time) { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write(body) + return + } + + allowed, err := h.allowedVersions(r.Context(), escapedModule, module) + if err != nil { + h.writeGoMetadataError(w, err) + return + } + best := newestGoVersion(allowed) + if best == "" { + h.refuseCooldown(w, module, info.Version) + return + } + escapedVersion, err := gomodule.EscapeVersion(best) + if err != nil { + h.refuseCooldown(w, module, info.Version) + return + } + h.serveCooldownInfo(w, r, escapedModule, module, escapedVersion+".info") +} + +// newestGoVersion returns the highest release in versions, or the highest +// pre-release when there is no release. +func newestGoVersion(versions []string) string { + var release, prerelease string + for _, v := range versions { + if !semver.IsValid(v) { + continue + } + if semver.Prerelease(v) == "" { + if release == "" || semver.Compare(v, release) > 0 { + release = v + } + } else if prerelease == "" || semver.Compare(v, prerelease) > 0 { + prerelease = v + } + } + if release != "" { + return release + } + return prerelease +} diff --git a/internal/handler/go_cooldown_test.go b/internal/handler/go_cooldown_test.go new file mode 100644 index 00000000..c5281957 --- /dev/null +++ b/internal/handler/go_cooldown_test.go @@ -0,0 +1,191 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +type goCooldownUpstream struct { + server *httptest.Server + mu sync.Mutex + hits map[string]int +} + +func (u *goCooldownUpstream) count(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +// newGoCooldownUpstream serves example.com/mod with an old v1.0.0, a +// recent v1.1.0 and a recent v1.2.0-rc.1, and @latest pointing at v1.1.0. +func newGoCooldownUpstream(t *testing.T, latest string) *goCooldownUpstream { + t.Helper() + now := time.Now() + times := map[string]time.Time{ + "v1.0.0": now.Add(-10 * 24 * time.Hour), + "v1.1.0": now.Add(-time.Hour), + "v1.2.0-rc.1": now.Add(-2 * time.Hour), + } + u := &goCooldownUpstream{hits: map[string]int{}} + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.hits[r.URL.Path]++ + u.mu.Unlock() + rest, ok := strings.CutPrefix(r.URL.Path, "/example.com/mod/") + if !ok { + http.NotFound(w, r) + return + } + writeInfo := func(v string) { + _ = json.NewEncoder(w).Encode(goInfo{Version: v, Time: times[v]}) + } + switch { + case rest == "@v/list": + _, _ = w.Write([]byte("v1.0.0\nv1.1.0\nv1.2.0-rc.1\n")) + case rest == "@latest": + writeInfo(latest) + case rest == "@v/main.info": + writeInfo("v1.1.0") + case strings.HasSuffix(rest, ".info"): + v := strings.TrimSuffix(strings.TrimPrefix(rest, "@v/"), ".info") + if _, ok := times[v]; !ok { + http.NotFound(w, r) + return + } + writeInfo(v) + case strings.HasSuffix(rest, ".mod"): + _, _ = w.Write([]byte("module example.com/mod\n")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.server.Close) + return u +} + +func goCooldownHandler(t *testing.T, u *goCooldownUpstream) (http.Handler, *mockFetcher, *Proxy) { + t.Helper() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes(), fetcher, proxy +} + +func goGet(h http.Handler, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w +} + +func TestGoCooldownFiltersList(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@v/list") + if w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + if got := w.Body.String(); got != "v1.0.0\n" { + t.Fatalf("list = %q, want only v1.0.0", got) + } + + // Publish times are stored, so a second list needs no .info requests. + _ = goGet(h, "/example.com/mod/@v/list") + if n := u.count("/example.com/mod/@v/v1.0.0.info"); n != 1 { + t.Errorf("v1.0.0.info fetched %d times, want 1", n) + } +} + +func TestGoCooldownRefusesVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, fetcher, _ := goCooldownHandler(t, u) + + for _, p := range []string{ + "/example.com/mod/@v/v1.1.0.info", + "/example.com/mod/@v/v1.1.0.mod", + "/example.com/mod/@v/v1.1.0.zip", + "/example.com/mod/@v/main.info", + } { + w := goGet(h, p) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errGoCooldown) { + t.Errorf("%s: status = %d body %q, want 403 cooldown", p, w.Code, w.Body.String()) + } + } + if fetcher.fetchCalled { + t.Error("zip in cooldown was fetched") + } + + for _, p := range []string{"/example.com/mod/@v/v1.0.0.info", "/example.com/mod/@v/v1.0.0.mod"} { + if w := goGet(h, p); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200", p, w.Code) + } + } +} + +func TestGoCooldownLatest(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@latest") + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + var info goInfo + if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil { + t.Fatal(err) + } + if info.Version != "v1.0.0" { + t.Fatalf("@latest = %s, want v1.0.0", info.Version) + } +} + +func TestGoCooldownLatestAllowed(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.0.0") + h, _, _ := goCooldownHandler(t, u) + + w := goGet(h, "/example.com/mod/@latest") + var info goInfo + if err := json.Unmarshal(w.Body.Bytes(), &info); err != nil || info.Version != "v1.0.0" { + t.Fatalf("@latest = %d %s, want v1.0.0", w.Code, w.Body) + } + if n := u.count("/example.com/mod/@v/list"); n != 0 { + t.Errorf("list fetched %d times for an allowed @latest", n) + } +} + +func TestGoCooldownDisabledPassesThrough(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + proxy.Cooldown = nil + + if w := goGet(h, "/example.com/mod/@v/list"); w.Body.String() != "v1.0.0\nv1.1.0\nv1.2.0-rc.1\n" { + t.Fatalf("list = %q", w.Body.String()) + } + if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusOK { + t.Fatalf("info status = %d", w.Code) + } +} + +func TestNewestGoVersion(t *testing.T) { + tests := []struct { + versions []string + want string + }{ + {[]string{"v1.0.0", "v1.10.0", "v1.9.0", "v2.0.0-rc.1"}, "v1.10.0"}, + {[]string{"v1.0.0-alpha", "v1.0.0-beta"}, "v1.0.0-beta"}, + {[]string{"v2.0.0+incompatible", "v1.5.0"}, "v2.0.0+incompatible"}, + {nil, ""}, + } + for _, tt := range tests { + if got := newestGoVersion(tt.versions); got != tt.want { + t.Errorf("newestGoVersion(%v) = %q, want %q", tt.versions, got, tt.want) + } + } +} From a7879137277536718ca3d5258865a717e82ac357 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 12:46:39 +0200 Subject: [PATCH 2/2] Fail closed in Go cooldown and key stored times by escaped path A failed .info fetch counted as an unknown time, so a rate-limited or flaky upstream let versions in cooldown through. It now fails the list or download with 502; only a version upstream doesn't know, or whose .info has no time, is let through. Times are stored under the escaped module path, which keeps modules that differ only in case apart. Modules with uppercase letters were not stored before and fetched every .info again on each list request. Only canonical versions are stored, since a query such as a branch name can resolve to a later commit. A .mod or .zip whose version doesn't unescape is answered with 404 instead of being passed upstream unchecked. --- docs/configuration.md | 2 +- internal/handler/go.go | 29 +++++---- internal/handler/go_cooldown.go | 89 +++++++++++++++++++--------- internal/handler/go_cooldown_test.go | 69 +++++++++++++++++++++ 4 files changed, 149 insertions(+), 40 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index eb99cd9e..d13ee876 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -982,7 +982,7 @@ Resolution order: exact package override, then package pattern, then ecosystem o Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. -Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. +Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/go.go b/internal/handler/go.go index 85cf7e9f..e544cd73 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -102,8 +102,12 @@ func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { case strings.HasSuffix(rest, ".mod"): // GET /{module}/@v/{version}.mod - go.mod file - if version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")); err == nil && h.versionInCooldown(r.Context(), module, decodedMod, version) { - h.refuseCooldown(w, decodedMod, version) + version, err := gomodule.UnescapeVersion(strings.TrimSuffix(rest, ".mod")) + if err != nil { + http.NotFound(w, r) + return + } + if h.checkCooldown(w, r, module, decodedMod, version) { return } h.proxyCached(w, r, decodedMod+"/@v/"+rest) @@ -145,15 +149,18 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul // The URL escapes uppercase letters in the version (v1.0.0-!r!c1 for // v1.0.0-RC1), and the zip is cached under that form, so the denylist // is checked against the version itself. - if unescaped, err := gomodule.UnescapeVersion(version); err == nil { - if h.proxy.versionDenied("golang", decodedModule, unescaped) { - http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) - return - } - if h.versionInCooldown(r.Context(), module, decodedModule, unescaped) { - h.refuseCooldown(w, decodedModule, unescaped) - return - } + // A version that doesn't unescape can't be checked, so it isn't fetched. + unescaped, err := gomodule.UnescapeVersion(version) + if err != nil { + http.NotFound(w, r) + return + } + if h.proxy.versionDenied("golang", decodedModule, unescaped) { + http.Error(w, ErrVersionDenied.Error()+": "+canonicalVersionPURL("golang", decodedModule, unescaped), http.StatusForbidden) + return + } + if h.checkCooldown(w, r, module, decodedModule, unescaped) { + return } downloadURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index f73a897f..c3aa0cbb 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "strings" "time" @@ -62,59 +63,85 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// storesGoTimes reports whether publish times for module can be kept in the -// versions table. Its rows are keyed by PURL, which lowercases golang names, -// so modules that differ only in case would share them; such modules read -// their times from the metadata cache instead. -func (h *GoHandler) storesGoTimes(module string) bool { - return h.proxy.DB != nil && module == strings.ToLower(module) +// goTimeKey returns the versions-table PURLs a module's times are stored +// under. golang PURLs lowercase the name, so the key uses the escaped module +// path (github.com/!azure for github.com/Azure), which keeps modules that +// differ only in case apart and is the plain path for lowercase modules. +// Only canonical versions are stored: a query such as a branch name can +// resolve to another commit later. +func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { + if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { + return "", "", false + } + escaped, err := gomodule.EscapePath(module) + if err != nil { + return "", "", false + } + return canonicalVersionPURL("golang", escaped, version), canonicalPackagePURL("golang", escaped), true } // publishTime returns a version's time from the versions table, or from its -// .info, which is then stored. A version whose time can't be read gets the -// zero time and is let through, as in the other ecosystems. -func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) time.Time { - versionPURL := canonicalVersionPURL("golang", module, version) - if h.storesGoTimes(module) { +// .info, which is then stored. A version upstream doesn't know, or whose +// .info has no time, gets the zero time and is let through, as in the other +// ecosystems. A failed fetch is an error, so a flaky or rate-limited +// upstream can't let a version through. +func (h *GoHandler) publishTime(ctx context.Context, escapedModule, module, version string) (time.Time, error) { + if versionPURL, _, ok := h.goTimeKey(module, version); ok { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return ver.PublishedAt.Time + return ver.PublishedAt.Time, nil } } escapedVersion, err := gomodule.EscapeVersion(version) if err != nil { - return time.Time{} + return time.Time{}, nil } body, err := h.fetchGoMetadata(ctx, escapedModule, module, "@v/"+escapedVersion+".info") + if errors.Is(err, ErrUpstreamNotFound) { + return time.Time{}, nil + } if err != nil { - h.proxy.Logger.Warn("cooldown: could not fetch go version info", - "module", module, "version", version, "error", err) - return time.Time{} + return time.Time{}, err } var info goInfo - if err := json.Unmarshal(body, &info); err != nil || info.Time.IsZero() { - return time.Time{} + if err := json.Unmarshal(body, &info); err != nil { + return time.Time{}, err } - h.storePublishTime(module, version, info.Time) - return info.Time + if !info.Time.IsZero() { + h.storePublishTime(module, version, info.Time) + } + return info.Time, nil } func (h *GoHandler) storePublishTime(module, version string, published time.Time) { - if !h.storesGoTimes(module) { + versionPURL, packagePURL, ok := h.goTimeKey(module, version) + if !ok { return } - if err := h.proxy.DB.SetVersionPublishedAt(canonicalVersionPURL("golang", module, version), canonicalPackagePURL("golang", module), published); err != nil { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { h.proxy.Logger.Warn("cooldown: could not store go publish time", "module", module, "version", version, "error", err) } } -// versionInCooldown reports whether an exact version is still in cooldown. -func (h *GoHandler) versionInCooldown(ctx context.Context, escapedModule, module, version string) bool { +// checkCooldown answers a request for an exact version in cooldown, or whose +// time couldn't be read, and reports whether it did. +func (h *GoHandler) checkCooldown(w http.ResponseWriter, r *http.Request, escapedModule, module, version string) bool { if !h.cooldownEnabled() { return false } - return !h.allowedAt(module, h.publishTime(ctx, escapedModule, module, version)) + published, err := h.publishTime(r.Context(), escapedModule, module, version) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not read go version time", + "module", module, "version", version, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + if !h.allowedAt(module, published) { + h.refuseCooldown(w, module, version) + return true + } + return false } func (h *GoHandler) refuseCooldown(w http.ResponseWriter, module, version string) { @@ -141,11 +168,17 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s g.SetLimit(goInfoFetchLimit) for i, v := range versions { g.Go(func() error { - keep[i] = h.allowedAt(module, h.publishTime(gctx, escapedModule, module, v)) + published, err := h.publishTime(gctx, escapedModule, module, v) + if err != nil { + return fmt.Errorf("reading time of %s: %w", v, err) + } + keep[i] = h.allowedAt(module, published) return nil }) } - _ = g.Wait() + if err := g.Wait(); err != nil { + return nil, err + } allowed := versions[:0] for i, v := range versions { @@ -186,7 +219,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es h.refuseCooldown(w, module, info.Version) return } - if !info.Time.IsZero() && semver.IsValid(info.Version) { + if !info.Time.IsZero() { h.storePublishTime(module, info.Version, info.Time) } } diff --git a/internal/handler/go_cooldown_test.go b/internal/handler/go_cooldown_test.go index c5281957..bedc3346 100644 --- a/internal/handler/go_cooldown_test.go +++ b/internal/handler/go_cooldown_test.go @@ -189,3 +189,72 @@ func TestNewestGoVersion(t *testing.T) { } } } + +func TestGoCooldownFailsClosedOnUpstreamErrors(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/@v/list"): + _, _ = w.Write([]byte("v1.0.0\n")) + case strings.HasSuffix(r.URL.Path, ".info"): + http.Error(w, "slow down", http.StatusTooManyRequests) + default: + _, _ = w.Write([]byte("module example.com/mod\n")) + } + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() + + for _, p := range []string{ + "/example.com/mod/@v/list", + "/example.com/mod/@v/v1.0.0.mod", + "/example.com/mod/@v/v1.0.0.zip", + } { + if w := goGet(h, p); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", p, w.Code) + } + } + if fetcher.fetchCalled { + t.Error("zip was fetched without a known time") + } +} + +func TestGoCooldownStoresTimesPerExactModule(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", u.server.URL).Routes() + + _ = goGet(h, "/example.com/mod/@v/list") + ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("time of v1.0.0 not stored: %v %v", ver, err) + } + + gh := &GoHandler{proxy: proxy} + upper, _, ok := gh.goTimeKey("github.com/Azure/sdk", "v1.0.0") + lower, _, _ := gh.goTimeKey("github.com/azure/sdk", "v1.0.0") + if !ok || upper == lower { + t.Errorf("modules differing in case share a key: %q %q", upper, lower) + } + for _, query := range []string{"master", "v1.2", "v1.0.0+meta"} { + if _, _, ok := gh.goTimeKey("example.com/mod", query); ok { + t.Errorf("time of non-canonical %q would be stored", query) + } + } +} + +func TestGoRejectsBadlyEscapedVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, fetcher, _ := goCooldownHandler(t, u) + for _, p := range []string{"/example.com/mod/@v/V1.0.0.zip", "/example.com/mod/@v/V1.0.0.mod"} { + if w := goGet(h, p); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", p, w.Code) + } + } + if fetcher.fetchCalled || u.count("/example.com/mod/@v/V1.0.0.mod") != 0 { + t.Error("badly escaped version reached upstream") + } +}