Serve conda sharded repodata through cooldown and the cache

README.md

@@ -411,6 +411,8 @@ Or set via command:
 conda config --add channels http://localhost:8080/conda/main
 ```
 
+pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on.
+
 ### CRAN (R)
 
 Set the repository in R:

docs/configuration.md

@@ -982,6 +982,8 @@ Resolution order: exact package override, then package pattern, then ecosystem o
 
 Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata.
 
+Conda cooldown covers `repodata.json` and sharded repodata (CEP 16). Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`.
+
 Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`.
 
 ## Version denylist

go.mod

@@ -24,11 +24,13 @@ require (
 	github.com/git-pkgs/vulns v0.2.4
 	github.com/go-chi/chi/v5 v5.5.0
 	github.com/jmoiron/sqlx v1.4.0
+	github.com/klauspost/compress v1.20.1
 	github.com/lib/pq v1.12.3
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/prometheus/client_golang v1.25.0
 	github.com/prometheus/client_model v0.6.3
 	github.com/swaggo/swag v1.16.6
+	github.com/vmihailenco/msgpack/v5 v5.4.1
 	gocloud.dev v0.47.0
 	golang.org/x/mod v0.42.0
 	golang.org/x/net v0.61.0
@@ -90,7 +92,6 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect
 	github.com/googleapis/gax-go/v2 v2.24.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.20.1 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.24 // indirect
@@ -108,6 +109,7 @@ require (
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
 	github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
 	github.com/ulikunitz/xz v0.5.17 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/otel v1.46.0 // indirect
 	go.opentelemetry.io/otel/metric v1.46.0 // indirect

go.sum

@@ -275,6 +275,10 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
 github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
 github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4=
 github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/detectors/gcp v1.46.0 h1:PI8dGkqDaQkwJ8kOopqMhDTbrnK3UIeG/RCHH4HErbo=

internal/handler/conda.go

@@ -82,6 +82,11 @@ func (h *CondaHandler) Routes() http.Handler {
 	mux.HandleFunc("GET /{channel}/{arch}/repodata.json.bz2", h.proxyCached)
 	mux.HandleFunc("GET /{channel}/{arch}/current_repodata.json", h.handleRepodata)
 
+	// Sharded repodata (CEP 16)
+	mux.HandleFunc("GET /{channel}/{arch}/"+condaShardIndexFile, h.handleShardIndex)
+	mux.HandleFunc("GET /{channel}/{arch}/shards/{filename}", h.handleShard)
+	mux.HandleFunc("GET /{channel}/{arch}/"+condaCooldownShardDir+"/{bucket}/{rest...}", h.handleCooldownShard)
+
 	// Package downloads (cache these)
 	mux.HandleFunc("GET /{channel}/{arch}/{filename}", h.handleDownload)
 
@@ -94,6 +99,11 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
 	arch := r.PathValue("arch")
 	filename := r.PathValue("filename")
 
+	if strings.HasSuffix(filename, condaShardSuffix) {
+		h.handleShard(w, r)
+		return
+	}
+
 	// Only cache actual package files
 	if !h.isPackageFile(filename) {
 		h.proxyUpstream(w, r)
@@ -288,22 +298,31 @@ type repodataEntry struct {
 // Entries without a name or timestamp are kept.
 func (h *CondaHandler) inCooldown(filename string, raw []byte) bool {
 	var entry repodataEntry
-	if err := json.Unmarshal(raw, &entry); err != nil || entry.Name == "" || entry.Timestamp == 0 {
+	if err := json.Unmarshal(raw, &entry); err != nil {
+		return false
+	}
+	return h.recordInCooldown(filename, entry.Name, entry.Version, entry.Timestamp)
+}
+
+// recordInCooldown reports whether a package record with a millisecond
+// timestamp was published too recently. Records without a name or
+// timestamp are kept.
+func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool {
+	if name == "" || timestamp == 0 {
 		return false
 	}
-	publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0)
-	if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", entry.Name), publishedAt) {
+	publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0)
+	if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) {
 		return false
 	}
 	h.proxy.Logger.Info("cooldown: filtering conda package",
-		"name", entry.Name, "version", entry.Version, "filename", filename)
+		"name", name, "version", version, "filename", filename)
 	return true
 }
 
 // proxyCached forwards a metadata request with caching.
 func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) {
-	cacheKey := strings.TrimPrefix(r.URL.Path, "/")
-	cacheKey = strings.ReplaceAll(cacheKey, "/", "_")
+	cacheKey := condaCacheKey(r.URL.Path)
 	// Large JSON indexes can exceed the metadata limit uncompressed. Conda
 	// clients decode Content-Encoding, so cache and relay gzip verbatim. The
 	// already-compressed repodata.json.bz2 route must stay on identity.