From cb86b96a21bf6ad6c9168be5bdd8d53a4673b959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammet=20=C5=9Eafak?= Date: Sun, 25 Oct 2026 08:46:14 +0300 Subject: [PATCH 1/2] Bound Pulsar and Dispatcher redelivery and route poison per ADR-0036 Consume\Dispatcher now caps attempts (default 3), routes poison to a .dlq derived from the consumer's own queue or topic (never default.dlq), drops and logs when no DLQ writer exists, and on Kafka falls back to the subscribed topic or degrades to commit + drop + log when the target is ambiguous. PulsarConsumer retries by re-publishing with bq-attempts + 1 through a producer derived from the client's service URL, so the cap holds on Exclusive and Failover subscriptions; DLQ records keep the bq-job route. Identity-less bodies are poison. Vendors conformance v1.7.0. --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 133 ++- README.md | 58 ++ src/Codec/StrictBodyClassifier.php | 29 + src/Consume/DeadLetterPublisher.php | 112 ++- src/Consume/DeadLetterWriteFailed.php | 22 + src/Consume/Dispatcher.php | 293 ++++++- src/Contracts/HasMessageProperties.php | 22 + src/Contracts/HasQueue.php | 20 + .../Internal/WebSocketConnection.php | 252 ++++++ src/Transport/KafkaConsumer.php | 48 +- src/Transport/KafkaConsumerClient.php | 9 +- src/Transport/KafkaMessage.php | 15 +- src/Transport/KafkaSubscriptionDescriptor.php | 21 + src/Transport/PulsarConsumer.php | 309 ++++++- src/Transport/PulsarConsumerDescriptor.php | 31 + src/Transport/PulsarMessage.php | 37 +- src/Transport/PulsarServiceEndpoint.php | 29 + src/Transport/PulsarTransport.php | 16 +- .../PulsarWebSocketConsumerClient.php | 6 +- src/Transport/PulsarWebSocketProducer.php | 87 ++ src/Transport/PulsarWebSocketSubscription.php | 217 +++++ tests/Consume/DeadLetterPublisherTest.php | 34 +- tests/Consume/DispatcherTerminationTest.php | 641 ++++++++++++++ tests/Consume/DispatcherTest.php | 15 +- tests/IdentityVerdictConformanceTest.php | 28 + tests/KafkaConsumerTest.php | 125 ++- tests/KafkaKeyPreservationTest.php | 9 +- tests/PulsarConsumerConformanceTest.php | 6 +- tests/PulsarConsumerTest.php | 35 +- tests/PulsarDerivedRetryProducerTest.php | 445 ++++++++++ tests/PulsarExhaustionConformanceTest.php | 143 +++ tests/PulsarRetryRepublishTest.php | 644 ++++++++++++++ tests/PulsarWebSocketSubscriptionTest.php | 238 +++++ tests/Support/pulsar_ws_server.php | 245 ++++++ tests/conformance/CONFORMANCE_VERSION | 2 +- tests/conformance/manifest.json | 819 +++++++++++++++++- tests/conformance/schema/results.example.json | 2 +- 39 files changed, 5081 insertions(+), 120 deletions(-) create mode 100644 src/Codec/StrictBodyClassifier.php create mode 100644 src/Consume/DeadLetterWriteFailed.php create mode 100644 src/Contracts/HasMessageProperties.php create mode 100644 src/Contracts/HasQueue.php create mode 100644 src/Transport/Internal/WebSocketConnection.php create mode 100644 src/Transport/KafkaSubscriptionDescriptor.php create mode 100644 src/Transport/PulsarConsumerDescriptor.php create mode 100644 src/Transport/PulsarServiceEndpoint.php create mode 100644 src/Transport/PulsarWebSocketProducer.php create mode 100644 src/Transport/PulsarWebSocketSubscription.php create mode 100644 tests/Consume/DispatcherTerminationTest.php create mode 100644 tests/IdentityVerdictConformanceTest.php create mode 100644 tests/PulsarDerivedRetryProducerTest.php create mode 100644 tests/PulsarExhaustionConformanceTest.php create mode 100644 tests/PulsarRetryRepublishTest.php create mode 100644 tests/PulsarWebSocketSubscriptionTest.php create mode 100644 tests/Support/pulsar_ws_server.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d73ad5..e6b3816 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ permissions: env: # The canonical conformance release this SDK is vendored from (ADR-0032). - BQ_CONFORMANCE_VERSION: "1.6.0" + BQ_CONFORMANCE_VERSION: "1.7.0" # sha256 of that release's fetch.sh: the script runs on the runner, so it is pinned rather than # trusted from a mutable tag. Bump together with BQ_CONFORMANCE_VERSION. BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 642acc7..f975ddd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest env: # The pinned conformance release (ADR-0032). Keep identical to ci.yml. - BQ_CONFORMANCE_VERSION: "1.6.0" + BQ_CONFORMANCE_VERSION: "1.7.0" BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849 steps: - uses: actions/checkout@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index c55b02d..7419b71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,133 @@ The envelope wire format is versioned separately by `meta.schema_version` ## [Unreleased] +Implements ADR-0036 decisions 3, 4 and 8 for the PHP core against conformance v1.7.0 (new sections +`identity_verdict` and `pulsar_exhaustion`). The envelope stays at `schema_version: 1`. This is a **MINOR** +with behaviour changes, listed first. + +### Behaviour changes +- **`Consume\Dispatcher` default `maxAttempts` is now `3` (was `0`, no cap).** The value is the total number of + deliveries a message gets, the first included; `0` is still accepted and means no cap. With a + `DeadLetterPublisher`, a handler that keeps throwing is now dead-lettered (`reason: "failed"`) at its third + delivery, and `on_unknown_urn: fail` / `release` of an unknown URN stop at the third delivery too (`reason: + "unknown_urn"`); before, both were redelivered without limit. A deployment that relied on no cap must pass + `maxAttempts: 0`. +- **Pulsar: a `Dispatcher` without a `DeadLetterPublisher` now acknowledges, drops and logs a message whose + handler keeps failing at the cap** (it used to redeliver it forever). Only a `PulsarMessage` is dropped: for + Kafka and for any other `ConsumedMessage` implementation the exception is still rethrown. The drop is logged through the new + `$onDrop` constructor argument (`fn(ConsumedMessage $message, string $why): void`); the default is + `error_log`, never with the body. +- **`on_unknown_urn: dead_letter` with no publisher now logs the drop** (unknown URN, and a poison body); it used + to ack silently. +- **Kafka (`Dispatcher` over `KafkaConsumer`): the `Dispatcher` never drops.** At the cap without a publisher it + rethrows (the `KafkaRetryRouter` or the consumer decides); with a publisher it dead-letters to `.dlq`. + With the new default cap, a `Dispatcher` with a publisher dead-letters a Kafka record at its third delivery, + before a `KafkaRetryRouter` with its default `maxAttempts` of 5. Pass `maxAttempts: 0` (or no publisher) to the + `Dispatcher` to leave the cap to the router. +- **Poison and identity-less bodies take their own route (ADR-0036 decisions 3 and 4).** A body that does not + decode, has a wrong-typed member, or has neither a valid `job` nor a valid `urn` (the `identity_verdict` + table, the same as `kafka_poison`) never reaches a handler and is no longer an "unknown URN": with a publisher + it is written to `.dlq` as the ADR-0035 record (`reason: "poison"`, no retry; before, a body with no + `meta.queue` went to `default.dlq`, and with `fail`/`release` it was redelivered without limit); `delete` + drops it; with no publisher `fail`/`release` take the bounded path above and `dead_letter` drops (Pulsar) or + rethrows (Kafka). A Pulsar body that carries `"data": []` (an empty JSON list instead of `{}`) is now poison, + as on Kafka. +- **`dead_letter.attempts` is the counted value** for the records the `Dispatcher` writes on exhaustion + (`failed`, and the new `unknown_urn` cap): the reconciled attempts plus one for the current delivery, the value + compared with the cap (it used to be the delivered value, one lower). A direct dead-letter (`on_unknown_urn: + dead_letter`, poison) keeps the delivered value. + +- **The DLQ is `.dlq`, never `default.dlq` (ADR-0036 decision 3).** The target comes from the + consumer's own queue / topic (`PulsarMessage::queue()`), not from the envelope's `meta.queue`: a producer may have + written any `meta.queue` (`EnvelopeCodec::make()` defaults it to `default`) onto a message that reached another + topic, and that name must not pick the DLQ. `dead_letter.original_queue` is the envelope's `meta.queue` when + present, else the consumer's queue. When the consumer's queue is unknown and a publisher IS set, the `Dispatcher` + applies the section 6.5 degrade on Pulsar and Kafka alike: acknowledge / commit, drop and log, no rethrow (a rethrow + would redeliver a record that can never be addressed and wedge the partition). With no publisher at all the earlier + rule stands (dropped and logged on Pulsar, rethrown on Kafka). Kafka follows the same rule: + `KafkaMessage` now implements `HasQueue` and carries the topic its `KafkaConsumerClient` reported (the optional + `topic` key of the `receive()` frame; `KafkaMessage::queue()` / the new 6th constructor argument). Behaviour + change for Kafka: over `KafkaConsumer` the DLQ used to be the envelope's `meta.queue` (so `meta.queue: "default"` + wrote `default.dlq`); it is now `.dlq`. The topic is the frame's `topic`, else `KafkaConsumer`'s `$topic` + argument, else the client's single subscribed topic (`KafkaSubscriptionDescriptor`); a multi-topic subscription with + no frame topic is ambiguous, and then the `Dispatcher` (with a publisher) commits, drops and logs the message instead of + rethrowing. A deployment that relied on `default.dlq` must pass `queue:` to `PulsarConsumer` / report the `topic` + from its `KafkaConsumerClient` (or use a client that reports its topic). Calling `DeadLetterPublisher::publish()` directly: a given `$queue` now wins over `meta.queue`, and a + message with neither throws a `BabelQueueException` instead of writing `default.dlq`. +- **The Pulsar DLQ record keeps the incoming route.** `PulsarTransport` now implements `HeaderPublisher` + (`publishWithHeaders()`, headers merged over the `bq-` projection; no headers is exactly `publish()`), and the + `DeadLetterPublisher` carries the incoming message properties (`bq-job`, ... except `bq-attempts` and + `bq-schema-version`) of a `PulsarMessage` onto the record, so a redrive routes it where the original was routed. +- **A failed dead-letter write is a nack, not a retry copy.** `DeadLetterPublisher::publish()` now throws + `Consume\DeadLetterWriteFailed` when the transport fails, and `PulsarConsumer::consume()` negatively acknowledges the + original (broker-bindings §5.5) instead of re-publishing it with `bq-attempts` + 1. +- **Pulsar retry is a re-publish (ADR-0036 decision 1, the Go model).** A negative acknowledgement advances + Pulsar's redelivery count only on `Shared` / `Key_Shared`; on `Exclusive` / `Failover` it does not, so the cap + was never reached. `PulsarConsumer::consume()` (new `PulsarConsumer::retry()`) now re-publishes a failed message + to the same topic with `bq-attempts` + 1 (an envelope's top-level `attempts` kept in sync; a poison or + version-gated body keeps its exact bytes and properties) and acknowledges the original, whatever the + subscription type; a failed re-publish negatively acknowledges the original. The consumer **derives** the + producer and the topic itself (explicit `retryProducer:` / `queue:` win for what they name): the producer is + the client when it also implements `PulsarWebSocketClient`; the topic comes from the new optional + `PulsarConsumerDescriptor` (`topic()`, `subscriptionType()`) on the client, else from the frame's `topic` key. + The copy goes to the **full topic name the message was read from** (`PulsarMessage::topic()`); a local name is + completed only with an explicit `tenant:` and `namespace:` (never a guessed `public/default`), and otherwise the + retry throws (a nack on `Shared` / `Key_Shared`). + When no producer can be derived the retry would be a nack, which is correct on `Shared` / `Key_Shared` only, so + the `PulsarConsumer` constructor **fails closed** (below). See the README section "Pulsar consumer: retry, queue + name and subscription type". +- **`new PulsarConsumer($client)` derives its retry producer, and fails closed only for a custom client that gives the SDK + nothing to derive it from.** A negative acknowledgement advances Pulsar's redelivery count only on + `Shared` / `Key_Shared`, and a `PulsarWebSocketConsumerClient` cannot ask the broker which type a subscription has, so an + unknown type is treated as `Exclusive` (Pulsar's default) and a nack-only retry would never reach `maxAttempts`. So + the producer is taken, in order, from: `retryProducer:`; the client itself when it also implements + `PulsarWebSocketClient`; or a `PulsarWebSocketProducer` on the Pulsar WebSocket service named by `serviceUrl:` + (token: `serviceAuthToken:`) or by the client's optional `PulsarServiceEndpoint` (`serviceUrl()`, `authToken()`); the + producer endpoint is the same host, `/ws/v2/producer/persistent///`, with tenant, namespace + and topic from the consumer's real full topic, never guessed. **The SDK's own client, the new + `PulsarWebSocketSubscription`, implements all of this: `new PulsarConsumer(new PulsarWebSocketSubscription($url, + $topic, $subscription))` needs no further argument and the cap holds on `Exclusive` / `Failover`.** Who is affected + (the constructor throws a `BabelQueueException` naming the way out): your own `PulsarWebSocketConsumerClient` + adapter that is none of the above, with no `retryProducer:`, no `serviceUrl:` and no declared `Shared` / + `Key_Shared` subscription; the SDK cannot know its service URL. The fix is one of: switch to + `PulsarWebSocketSubscription`; pass `serviceUrl:` (and `serviceAuthToken:` on an authenticated cluster) or let the + adapter implement `PulsarServiceEndpoint`; pass `retryProducer:`; or pass `subscriptionType: 'Shared'` (or + `'Key_Shared'`) if that is what the subscription is, which keeps the nack retry. +- **A derived producer must know the topic it re-publishes to, at setup.** On an `Exclusive` / `Failover` / unknown + subscription the constructor now also throws when the retry topic cannot be known: the client reports no full topic + (`PulsarConsumerDescriptor::topic()`), and `$queue` + `$tenant` + `$namespace` were not all given. If your adapter puts the + FULL topic name in every frame's `topic` key, say so with `topicFromFrame: true`. (Before, the first failing handler + would throw out of `consume()` instead.) A known `Shared` / `Key_Shared` subscription never needs the topic. +- A poison body that the lenient codec can still decode (for example a wrong-typed `trace_id`) no longer lends + its body `attempts` to the count: a poison body counts `bq-attempts` and the redelivery count only. + +### Added +- `Consume\DeadLetterWriteFailed` is no longer `final`, so the Laravel adapter can extend it later. +- `Transport\PulsarWebSocketSubscription`: the SDK's own Pulsar WebSocket consumer client (receive / acknowledge / + negative acknowledge over `/ws/v2/consumer/persistent////`, base64 payload + decoded, a read window that returns null when idle, reconnect on the next call after a broken connection). It + implements `PulsarConsumerDescriptor` and `PulsarServiceEndpoint`, so `new PulsarConsumer($subscription)` just works. +- `Transport\KafkaSubscriptionDescriptor` (optional, on a `KafkaConsumerClient`: `subscribedTopics()`) and a trailing `$topic` argument + on `KafkaConsumer`: where the consumer's topic comes from when a record frame carries none (see "Behaviour changes"). +- `Transport\PulsarWebSocketProducer`: a dependency-free Pulsar WebSocket producer (RFC 6455 over `stream_socket_client`, + `ssl://` for `wss` / `https`, optional bearer token): one `publish()` opens the producer endpoint of the full topic, + sends one frame and waits for `{"result":"ok"}`; any other outcome throws. Optional `Transport\PulsarServiceEndpoint` + interface (`serviceUrl()`, `authToken()`) and a trailing `serviceUrl:` argument on `PulsarConsumer`; no existing + signature changed. +- `PulsarConsumer` takes the queue it reads (`new PulsarConsumer($client, queue: 'orders')`, the topic's local + name), or the client adapter adds an optional `topic` key to the frame `receive()` returns, plus the optional + `retryProducer`, `tenant` (default null), `namespace` (default null) and `subscriptionType` arguments for the retry re-publish, and the optional + `PulsarConsumerDescriptor` interface a client adapter implements to report its topic and subscription type. The queue is exposed as + `PulsarMessage::queue()` (new `Contracts\HasQueue`; the message also exposes `properties()` through the new + `Contracts\HasMessageProperties`, and `topic()`). +- `Codec\StrictBodyClassifier::isPoison()`, the binding-independent name for the strict body verdict + (`identity_verdict`); it delegates to `KafkaBodyClassifier`. +- `DeadLetterPublisher::publish()` takes optional `$attempts` (the `dead_letter.attempts` value) and `$queue` + (used when the message has no `meta.queue`); `DeadLetterPublisher::resolveQueue()` says which queue a message + would use, or null when none can be named. +- Conformance runners for `identity_verdict` (97 cases) and `pulsar_exhaustion` (29 cases); the vendored suite + and the workflow pins move to conformance v1.7.0. + ## [1.25.0] - 2026-10-24 Implements ADR-0033 / ADR-0034 for the PHP core against conformance v1.6.0. The envelope stays at @@ -44,8 +171,10 @@ Implements ADR-0033 / ADR-0034 for the PHP core against conformance v1.6.0. The is written to `.dlq` with `dead_letter.reason = poison` (original key and headers preserved) and then committed; without a producer it degrades to commit + drop (section 6.5). `receive()` returns `null` for such a record. A failing DLQ write does not commit and the exception propagates, so the record is redelivered. - `KafkaConsumerClient::receive()` may now return an optional `topic` (fallback: body `meta.queue`, then - `default`). + `KafkaConsumerClient::receive()` may now return an optional `topic`. (Superseded in this release: the + `meta.queue` / `default` fallback is gone, a poison record from a client that reports no topic makes + `KafkaConsumer` throw `BabelQueueException` and commit nothing, so the partition does not silently feed + `default.dlq`; see "The DLQ is `.dlq`" above.) - `SqsTransport::MAX_VISIBILITY_TIMEOUT_SECONDS` (43 200). - `Consume\Dispatcher` takes an optional `$onRetry` closure `(ConsumedMessage, int $delaySeconds, Throwable)`, called when a handler throws and the message is not dead-lettered, so a transport release can apply the diff --git a/README.md b/README.md index d5479ee..9a5fff0 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,64 @@ else is rejected without decoding. A rejected record reaches the `Dispatcher` as `babelqueue:unsupported-schema-version:` (`KafkaConsumer::UNSUPPORTED_SCHEMA_URN_PREFIX`), so it takes the unknown-URN path (never a handler) and its dead-letter copy keeps the original bytes. +## Pulsar consumer: retry, queue name and subscription type + +`PulsarConsumer` + `Consume\Dispatcher` bound every message (`maxAttempts`, default 3; ADR-0036). A failing +handler is retried by **re-publishing** the message to the same topic with `bq-attempts` + 1 and acknowledging the +original, not by a negative acknowledgement: on an `Exclusive` / `Failover` subscription a nack does not advance +Pulsar's redelivery count, so a nack-only retry would never reach the cap. A poison or version-gated body keeps its +exact bytes and properties on the way. + +The consumer derives what it needs from your `PulsarWebSocketConsumerClient` adapter, and anything you pass +explicitly wins; an adapter that implements the two interfaces below needs no configuration at all. + +| It needs | Derived from | Explicit | +| :--- | :--- | :--- | +| the retry producer | the client itself, when it also implements `PulsarWebSocketClient`; else a `PulsarWebSocketProducer` on the service URL the client reports (`PulsarServiceEndpoint::serviceUrl()`) | `retryProducer:`, or `serviceUrl:` | +| the topic / queue (`.dlq`, the re-publish topic) | the client, when it implements `PulsarConsumerDescriptor` (`topic()`), else the `topic` key of the frame `receive()` returns (declare `topicFromFrame: true`). Give the **full** name (`persistent://tenant/ns/topic`): the retry copy goes to exactly that topic | `queue:` names the DLQ; a local name needs `tenant:` and `namespace:` to build the re-publish topic (never guessed) | +| the subscription type | `PulsarConsumerDescriptor::subscriptionType()` | `subscriptionType:` | + +```php +final class MyPulsarConsumerClient implements PulsarWebSocketConsumerClient, PulsarWebSocketClient, PulsarConsumerDescriptor +{ + // receive / acknowledge / negativeAcknowledge, publish(), then: + public function topic(): string { return 'persistent://public/default/orders'; } + public function subscriptionType(): ?string { return 'Exclusive'; } +} + +$consumer = new PulsarConsumer(new MyPulsarConsumerClient()); +$consumer->consume((new Dispatcher(deadLetters: new DeadLetterPublisher($transport)))->on($urn, $handler)); +``` + +**Fail-closed.** If no producer can be derived, the `PulsarConsumer` constructor throws a `BabelQueueException` +unless the subscription is *known* to be `Shared` / `Key_Shared` (there a nack advances the redelivery count, so a +nack-only retry works). An unknown subscription type counts as `Exclusive` (Pulsar's own default): this seam cannot ask +the broker, so the consumer refuses to start rather than run a retry that may never end. To keep a nack-only consumer, +declare it: `new PulsarConsumer($client, subscriptionType: 'Shared')` (or have the client implement +`PulsarConsumerDescriptor`). Otherwise give it a producer (`retryProducer:`, `serviceUrl:`, or a client that also implements +`PulsarWebSocketClient` / `PulsarServiceEndpoint`). A retry whose topic is still unknown throws as well, unless the subscription is known to be +`Shared` / `Key_Shared`. Nothing is ever written to `default.dlq`: the DLQ is `.dlq` (never picked from the +envelope's `meta.queue`), and when the queue is unknown the `Dispatcher` treats a message that needs the DLQ as having +no publisher (dropped and logged on Pulsar, rethrown on Kafka); a publisher that is set but cannot name the queue is dropped and logged on both. A `KafkaConsumerClient` reports the topic in the `topic` key of its `receive()` frame (else `KafkaConsumer`'s `$topic` argument, else the client's single `KafkaSubscriptionDescriptor` topic); `KafkaMessage::queue()` carries it. When it is unknown (a multi-topic subscription), a poison record is committed and dropped with a log line (section 6.5; the partition is never wedged) and so does the `Dispatcher` when it has a publisher (a rethrow would redeliver a record that can never be addressed); only with no publisher at all does it rethrow on Kafka. + +**Out of the box: `PulsarWebSocketSubscription`.** The SDK's own consumer client reads a subscription over +`/ws/v2/consumer/...` and reports its topic, subscription type, service URL and token, so this is the whole setup, and +the cap holds on `Exclusive` / `Failover` (the retry producer is built on the same service, from the consumer's real +tenant / namespace / topic): + +```php +$consumer = new PulsarConsumer(new PulsarWebSocketSubscription('http://pulsar:8080', 'persistent://acme/ns1/orders', 'my-sub')); +``` + +**Your own client that only reads.** A Pulsar WebSocket consumer connection cannot publish, but the producer endpoint is +on the same service (`/ws/v2/producer/persistent///`). So when your client implements +`PulsarServiceEndpoint` (`serviceUrl()`, `authToken()`), or you pass `serviceUrl:` (and `serviceAuthToken:` on an +authenticated cluster), the consumer builds a `PulsarWebSocketProducer` there itself. `retryProducer:` and a client that is +itself a `PulsarWebSocketClient` take precedence. Only a client that offers none of these (the SDK cannot know its service +URL) still fails closed: pass `serviceUrl:`, `retryProducer:`, or declare `Shared` / `Key_Shared`. The producer also needs a +topic to publish to: report a full one (`PulsarConsumerDescriptor::topic()`), pass `queue:` with `tenant:` and `namespace:`, +or declare `topicFromFrame: true` when every frame carries the full `topic`. + ## Design This core is the **contract runtime**, not a worker. It does not own a broker diff --git a/src/Codec/StrictBodyClassifier.php b/src/Codec/StrictBodyClassifier.php new file mode 100644 index 0000000..9278d24 --- /dev/null +++ b/src/Codec/StrictBodyClassifier.php @@ -0,0 +1,29 @@ +.dlq`, annotated with why it failed. + * Publish $message to `.dlq`, annotated with why it failed. + * + * **The target is the caller's `$queue`** (the consumer's own queue / topic, ADR-0036 decision 3), not the + * envelope's `meta.queue`: a producer may have written any `meta.queue` (`EnvelopeCodec::make()` defaults it to + * `default`) onto a message that reached another topic. Without a `$queue` the message's own `meta.queue` is + * used (callers that have no consumer queue, e.g. the Kafka consumers). `dead_letter.original_queue` is the + * envelope's `meta.queue` when present, else the target. **Nothing is ever written to `default.dlq`:** when + * neither names a queue, see {@see self::resolveQueue()}, the call throws. + * + * When the message carries its incoming native properties ({@see HasMessageProperties}, Pulsar) and the + * transport can take them ({@see HeaderPublisher}), the routing properties (`bq-job`, ...) ride on the record + * (all but `bq-attempts` and `bq-schema-version`, which the record's own body defines), so a redrive routes it + * where the original was routed. * * @param string $reason `failed` | `unknown_urn` | `poison` + * @param int|null $attempts the value for `dead_letter.attempts`; null = the message's delivered + * {@see ConsumedMessage::attempts()}. A retry-exhaustion caller passes the + * counted value (delivered + 1, the one compared with the cap; ADR-0036 decision 8) + * @param string|null $queue the queue the record goes to (the consumer's topic); null = the message's `meta.queue` + * + * @throws BabelQueueException When no queue can be named + * @throws DeadLetterWriteFailed When the transport fails to write the record */ - public function publish(ConsumedMessage $message, string $reason, ?Throwable $e): void + public function publish(ConsumedMessage $message, string $reason, ?Throwable $e, ?int $attempts = null, ?string $queue = null): void { - $queue = $message->getMeta()['queue'] ?? 'default'; - $queue = is_string($queue) && $queue !== '' ? $queue : 'default'; + $attempts ??= $message->attempts(); + $target = $this->resolveQueue($message, $reason, $queue) + ?? throw new BabelQueueException('Cannot name the dead-letter queue: the caller gave no queue and the message has no usable meta.queue (nothing is written to default.dlq).'); - $poisonRaw = $this->poisonRaw($message); + $poisonRaw = $this->poisonRaw($message, $reason); - // A poison copy is written byte-for-byte by PoisonBody (it cannot fail on the body: invalid UTF-8, - // 1e400, ...); encode() would reshape a rejected list "data" into an object. - $body = $poisonRaw !== null - ? PoisonBody::annotate($poisonRaw, $reason, $e, $queue, $message->attempts()) - : EnvelopeCodec::encode(DeadLetter::annotate($message->envelope(), $reason, $e, $queue, $message->attempts())); + if ($poisonRaw !== null) { + // A poison copy is written byte-for-byte by PoisonBody (it cannot fail on the body: invalid UTF-8, + // 1e400, ...); encode() would reshape a rejected list "data" into an object. + $body = PoisonBody::annotate($poisonRaw, $reason, $e, $target, $attempts); + } else { + $metaQueue = $message->getMeta()['queue'] ?? null; + $original = is_string($metaQueue) && $metaQueue !== '' ? $metaQueue : $target; + $body = EnvelopeCodec::encode(DeadLetter::annotate($message->envelope(), $reason, $e, $original, $attempts)); + } + + try { + $this->write($message, $body, $target . $this->suffix); + } catch (Throwable $writeError) { + throw DeadLetterWriteFailed::forQueue($target . $this->suffix, $writeError); + } + } + private function write(ConsumedMessage $message, string $body, string $destination): void + { // Carry the original broker record key (Kafka) onto the DLQ copy when there is one and the // transport can take it; a null key, or a transport without the capability, publishes as before. $key = $message instanceof HasRecordKey ? $message->recordKey() : null; if ($key !== null && $this->transport instanceof KeyedPublisher) { - $this->transport->publishKeyed($body, $queue . $this->suffix, $key); + $this->transport->publishKeyed($body, $destination, $key); return; } - $this->transport->publish($body, $queue . $this->suffix); + if ($message instanceof HasMessageProperties && $this->transport instanceof HeaderPublisher) { + $carried = $message->properties(); + unset($carried['bq-attempts'], $carried['bq-schema-version']); + + if ($carried !== []) { + $this->transport->publishWithHeaders($body, $carried, $destination); + + return; + } + } + + $this->transport->publish($body, $destination); } /** - * The raw body to dead-letter as a poison copy. A poison message (its body decodes to the empty - * envelope, e.g. malformed JSON or a list `data` that decode rejects) keeps an inspectable copy when - * the message carries its raw body ({@see HasRawBody}); see {@see PoisonBody}. + * The queue `$message` is dead-lettered to, or null when it cannot be named: the caller's `$queue` (the + * consumer's topic) when given; else, for a message that is not a poison copy (see {@see self::poisonRaw()}, + * which has no `meta.queue` of its own to trust), its own `meta.queue`. A null result is treated like "no + * publisher": the {@see Dispatcher} drops (Pulsar) or rethrows (Kafka) instead of publishing. + */ + public function resolveQueue(ConsumedMessage $message, string $reason, ?string $queue = null): ?string + { + if ($queue !== null && $queue !== '') { + return $queue; + } + + if ($this->poisonRaw($message, $reason) !== null) { + return null; + } + + $metaQueue = $message->getMeta()['queue'] ?? null; + + return is_string($metaQueue) && $metaQueue !== '' ? $metaQueue : null; + } + + /** + * The raw body to dead-letter as a poison copy. A poison message (reason `poison`, or a body that + * decodes to the empty envelope, e.g. malformed JSON or a list `data` that decode rejects) keeps an + * inspectable copy when the message carries its raw body ({@see HasRawBody}); see {@see PoisonBody}. + * A `poison` copy also keeps the bytes of a body that decodes to a non-empty envelope but that + * {@see StrictBodyClassifier} rejects (a wrong-typed member such as `data: [1, 2]`): its bytes must + * be kept (ADR-0035). * * Null when the message is not poison (or carries no raw body): its decoded envelope is used. */ - private function poisonRaw(ConsumedMessage $message): ?string + private function poisonRaw(ConsumedMessage $message, string $reason): ?string { if (! $message instanceof HasRawBody) { return null; } $raw = $message->rawBody(); - if ($raw === null || EnvelopeCodec::decode($raw, static function (): void {}) !== []) { + if ($raw === null) { return null; } - return $raw; + if (EnvelopeCodec::decode($raw, static function (): void {}) === []) { + return $raw; + } + + return $reason === 'poison' && StrictBodyClassifier::isPoison($raw) ? $raw : null; } } diff --git a/src/Consume/DeadLetterWriteFailed.php b/src/Consume/DeadLetterWriteFailed.php new file mode 100644 index 0000000..19b5190 --- /dev/null +++ b/src/Consume/DeadLetterWriteFailed.php @@ -0,0 +1,22 @@ +getMessage()), 0, $previous); + } +} diff --git a/src/Consume/Dispatcher.php b/src/Consume/Dispatcher.php index ff6d5ba..d91db4a 100644 --- a/src/Consume/Dispatcher.php +++ b/src/Consume/Dispatcher.php @@ -5,8 +5,14 @@ namespace BabelQueue\Consume; use BabelQueue\Contracts\ConsumedMessage; +use BabelQueue\Contracts\HasQueue; +use BabelQueue\Contracts\HasRawBody; use BabelQueue\Exceptions\UnknownUrnException; use BabelQueue\Routing\UnknownUrnStrategy; +use BabelQueue\Codec\StrictBodyClassifier; +use BabelQueue\Transport\KafkaMessage; +use BabelQueue\Transport\PulsarConsumer; +use BabelQueue\Transport\PulsarMessage; use Throwable; /** @@ -23,24 +29,62 @@ * $consumer->consume($dispatch, $shouldStop); * * It uses the consumer loop's **ack-on-return / redeliver-on-throw** contract: a handler that - * returns normally acks the message; a thrown handler redelivers it (at-least-once). On an unknown - * URN it applies the strategy — `delete` acks and drops, `dead_letter` routes to `.dlq` then - * acks (degrading to `delete` when no publisher is set), and `fail`/`release` throw to redeliver - * (delayed `release` is a binding-native / retry-topic concern, see the §x.5 bindings). When - * `maxAttempts > 0` and a publisher is set, a handler that keeps throwing is dead-lettered once the - * reconciled attempt count would reach the cap, instead of redelivering forever. + * returns normally acks the message; a thrown handler redelivers it (at-least-once). + * + * **Every message terminates (ADR-0036 decision 3).** The delivery is *counted* as + * `attempts() + 1` (the reconciled attempt count plus the current delivery) and compared with + * `$maxAttempts` (default **3**; `0` = no cap). A retryable outcome — a handler that throws, an unknown + * URN under `fail`/`release`, a poison body with no publisher — is redelivered (the exception is + * rethrown) while `counted < maxAttempts`. At the cap the message is **exhausted**: + * + * - with a {@see DeadLetterPublisher}: written to `.dlq` (`reason` `failed` / `unknown_urn`, + * `dead_letter.attempts` = the counted value); + * - without one, on **Pulsar** ({@see PulsarMessage}): acknowledged, dropped and logged; + * - without one, on **Kafka** ({@see KafkaMessage}) or any other message type: the exception is + * rethrown. The `Dispatcher` drops only a {@see PulsarMessage}; over Kafka the + * {@see \BabelQueue\Transport\KafkaRetryRouter} or the consumer decides (a drop there would commit + * the offset and lose the record), and for an unknown message type the previous behaviour is kept. **Note for Kafka users:** the default cap + * (3) dead-letters a record with a publisher at its third delivery, before a `KafkaRetryRouter` with + * its default `maxAttempts` of 5; pass `maxAttempts: 0` (or no publisher) to leave the cap to the router. + * + * **Retry is the consumer's re-publish.** The `Dispatcher` signals "retry" by rethrowing; on Pulsar the + * {@see \BabelQueue\Transport\PulsarConsumer} (given a `$retryProducer`) turns that into a re-publish with + * `bq-attempts` + 1 and an ack, so the cap is reached on every subscription type, `Exclusive` / `Failover` + * included, where a negative acknowledgement does not advance Pulsar's redelivery count (ADR-0036 decision 1). + * + * **An unnameable queue.** Nothing is ever written to `default.dlq` (ADR-0036 decision 3): the DLQ is + * `.dlq` with the queue taken from the consumer's topic ({@see HasQueue}: `PulsarConsumer`'s `queue:` + * argument or the frame topic, `KafkaConsumer`'s frame / `$topic` / single subscribed topic), never from the + * message's `meta.queue`. When a {@see DeadLetterPublisher} IS set but the consumer's queue is unknown (for example a + * multi-topic Kafka subscription), the section 6.5 degrade applies on Pulsar AND Kafka: the message is committed or + * acknowledged, dropped and logged, and the `Dispatcher` does not rethrow (a rethrow would redeliver a record that can + * never be addressed and wedge the partition). With NO publisher the rule above is unchanged: dropped and logged on + * Pulsar, rethrown on Kafka. + * + * **Unknown URN** (a valid envelope with no handler): `delete` acks and drops; `dead_letter` writes + * `.dlq` (`reason: unknown_urn`) then acks — with no publisher it acks, drops and **logs** (rethrows + * on Kafka, unless a publisher is set whose queue is unknown); `fail`/`release` are bounded as above. + * + * **Poison** (a body that does not decode into an envelope, or has no valid `job`/`urn` identity, + * ADR-0036 decision 4) never reaches a handler and has its own route, not the unknown-URN one: with a + * publisher (and any strategy but `delete`) the ADR-0035 record goes to `.dlq` with + * `reason: "poison"` and **no retry**, the queue being the consumer's topic ({@see HasQueue}); `delete` + * acks and drops; with no publisher `fail`/`release` take the bounded path above (a poison body is + * never released forever) and `dead_letter` acks, drops and logs (rethrows on Kafka). + * + * A drop is logged through `$onDrop($message, $why)` (default: `error_log`); it never carries the body. * * **Retry backoff (ADR-0034 §9, opt-in).** Hand it a {@see RetryBackoff} and a consumer that releases * a failed message with a delay asks {@see self::retryDelaySeconds()} for the seconds to pass to its * transport. To have the dispatcher apply it on the consume path, pass an `$onRetry` closure: whenever a - * handler throws and the message is *not* dead-lettered, `$onRetry($message, $delaySeconds, $e)` runs - * (before the exception is rethrown for redelivery), e.g. + * handler throws and the message is *not* dead-lettered or dropped, `$onRetry($message, $delaySeconds, $e)` + * runs (before the exception is rethrown for redelivery), e.g. * `fn ($m, $delay) => $sqs->release($handleOf($m), $delay)` — `SqsTransport::release()` clamps an * unbounded delay to SQS's 12 h limit. Without a backoff (the default) the delay is 0 — the previous * behaviour. Not applied on Kafka. * * Note on Kafka: `attempts` only grows across redeliveries when retry topics increment `bq-attempts` - * (§6.4/§6.5), so the `maxAttempts` dead-letter cap is effective on Pulsar (native redelivery count) + * (§6.4/§6.5), so the `maxAttempts` cap is effective on Pulsar (native redelivery count) * and on Kafka once retry topics are configured. */ final class Dispatcher @@ -48,12 +92,20 @@ final class Dispatcher /** @var array */ private array $handlers = []; + /** + * @param string $onUnknownUrn one of {@see UnknownUrnStrategy} + * @param int $maxAttempts total deliveries a message gets, the first included (default 3); 0 = no cap + * @param DeadLetterPublisher|null $deadLetters where exhausted / poison messages go; null = dead-lettering off + * @param \Closure|null $onDrop `fn(ConsumedMessage $message, string $why): void`, called whenever the + * Dispatcher acknowledges and drops a message; null = log via `error_log` + */ public function __construct( private readonly string $onUnknownUrn = UnknownUrnStrategy::FAIL, - private readonly int $maxAttempts = 0, + private readonly int $maxAttempts = 3, private readonly ?DeadLetterPublisher $deadLetters = null, private readonly ?RetryBackoff $retryBackoff = null, private readonly ?\Closure $onRetry = null, + private readonly ?\Closure $onDrop = null, ) { } @@ -90,6 +142,12 @@ public function on(string $urn, callable $handler): self */ public function __invoke(ConsumedMessage $message): void { + if ($this->isPoison($message)) { + $this->handlePoison($message); + + return; + } + $handler = $this->handlers[$message->getUrn()] ?? null; if ($handler === null) { @@ -101,12 +159,21 @@ public function __invoke(ConsumedMessage $message): void try { $handler($message); } catch (Throwable $e) { - if ($this->shouldDeadLetter($message)) { - /** @var DeadLetterPublisher $publisher */ - $publisher = $this->deadLetters; - $publisher->publish($message, 'failed', $e); + if ($this->exhausted($message)) { + if ($this->canDeadLetter($message, 'failed')) { + $this->deadLetter($message, 'failed', $e, $this->counted($message)); - return; + return; + } + + if ($message instanceof PulsarMessage || $this->isUnaddressable($message)) { + $this->drop($message, 'the handler kept failing and no dead-letter publisher can name its queue'); + + return; + } + // Kafka without ANY publisher (and any other type) never drops: rethrow below so the retry router / + // consumer decides. With a publisher whose queue is unknown, ADR-0036 decision 3 says commit, drop, log: + // a rethrow would redeliver a record that can never be addressed and wedge the partition. } if ($this->onRetry !== null) { @@ -117,32 +184,210 @@ public function __invoke(ConsumedMessage $message): void } } - private function shouldDeadLetter(ConsumedMessage $message): bool + /** + * The delivery counted against the cap: the reconciled attempts plus the current delivery. + */ + private function counted(ConsumedMessage $message): int + { + return $message->attempts() + 1; + } + + /** + * True when this delivery reaches the cap (`counted >= maxAttempts`); never with `maxAttempts` 0. + */ + private function exhausted(ConsumedMessage $message): bool + { + return $this->maxAttempts > 0 && $this->counted($message) >= $this->maxAttempts; + } + + /** + * A body that cannot be handed to a handler (ADR-0036 decision 4): no routing identity, or a raw + * body the strict classifier does not accept (undecodable, a wrong-typed member, no `job`/`urn`). + * The version-gate sentinel is not poison: it is an unknown URN by design. A Kafka record was + * already classified by the consumer, so only its identity is checked here. + */ + private function isPoison(ConsumedMessage $message): bool { - return $this->maxAttempts > 0 - && $this->deadLetters !== null - && $message->attempts() + 1 >= $this->maxAttempts; + $urn = $message->getUrn(); + if (str_starts_with($urn, PulsarConsumer::UNSUPPORTED_SCHEMA_URN_PREFIX)) { + return false; + } + + if ($urn === '') { + return true; + } + + if ($message instanceof KafkaMessage || ! $message instanceof HasRawBody) { + return false; + } + + $raw = $message->rawBody(); + + return $raw !== null && StrictBodyClassifier::isPoison($raw); + } + + private function handlePoison(ConsumedMessage $message): void + { + switch ($this->onUnknownUrn) { + case UnknownUrnStrategy::DELETE: + $this->drop($message, 'poison body, on_unknown_urn is delete'); + + return; + + case UnknownUrnStrategy::DEAD_LETTER: + if ($this->canDeadLetter($message, 'poison')) { + $this->deadLetter($message, 'poison', null); + + return; + } + + $this->dropOrRethrow($message, 'poison body, on_unknown_urn is dead_letter and no dead-letter publisher can name its queue', null, true); + + return; + + case UnknownUrnStrategy::RELEASE: + case UnknownUrnStrategy::FAIL: + default: + if ($this->canDeadLetter($message, 'poison')) { + $this->deadLetter($message, 'poison', null); // no retry: the same bytes would fail again + + return; + } + + // No publisher: the bounded `fail` path (a poison body is never released forever). + if ($this->exhausted($message)) { + $this->dropOrRethrow($message, 'poison body, attempts exhausted and no dead-letter publisher can name its queue'); + + return; + } + + throw new UnknownUrnException('The message body is poison: it does not decode into an envelope with a job or urn.'); + } } private function handleUnknownUrn(ConsumedMessage $message): void { switch ($this->onUnknownUrn) { case UnknownUrnStrategy::DELETE: - return; // ack + drop + return; // ack + drop (the explicit strategy: nothing to log) case UnknownUrnStrategy::DEAD_LETTER: - if ($this->deadLetters !== null) { - $this->deadLetters->publish($message, 'unknown_urn', null); + if ($this->canDeadLetter($message, 'unknown_urn')) { + $this->deadLetter($message, 'unknown_urn', null); + + return; } - return; // ack (degrades to delete when no publisher is configured) + // Dead-lettering is off: the strategy degrades to delete, visibly. + $this->dropOrRethrow($message, sprintf('no handler for URN [%s], on_unknown_urn is dead_letter and no dead-letter publisher can name its queue', $message->getUrn()), null, true); + + return; case UnknownUrnStrategy::RELEASE: case UnknownUrnStrategy::FAIL: default: - throw new UnknownUrnException( + $e = new UnknownUrnException( sprintf('No handler is registered for URN [%s].', $message->getUrn()), ); + + if (! $this->exhausted($message)) { + throw $e; + } + + if ($this->canDeadLetter($message, 'unknown_urn')) { + $this->deadLetter($message, 'unknown_urn', $e, $this->counted($message)); + + return; + } + + $this->dropOrRethrow($message, sprintf('no handler for URN [%s], attempts exhausted and no dead-letter publisher can name its queue', $message->getUrn()), $e); + } + } + + /** + * Write the DLQ record. `$attempts` is the counted value for a retry-exhaustion record, null for a + * direct dead-letter (the delivered attempts, informational). + */ + private function deadLetter(ConsumedMessage $message, string $reason, ?Throwable $e, ?int $attempts = null): void + { + /** @var DeadLetterPublisher $publisher */ + $publisher = $this->deadLetters; + $publisher->publish($message, $reason, $e, $attempts, $this->queueOf($message)); + } + + /** + * Whether a record for `$message` can be written: a publisher is set AND the queue is known (the consumer's + * for a {@see HasQueue} message). Nothing is ever written to `default.dlq`; an unnameable queue is handled + * like "no publisher". + */ + private function canDeadLetter(ConsumedMessage $message, string $reason): bool + { + // A message that knows its consumer (Pulsar, Kafka) is addressed by the consumer's queue only: a producer's + // `meta.queue` (e.g. the `default` that `EnvelopeCodec::make()` writes) must not pick the DLQ. A message of a + // type that does not carry one keeps using its own `meta.queue`. + if ($message instanceof HasQueue && $this->queueOf($message) === null) { + return false; } + + return $this->deadLetters !== null + && $this->deadLetters->resolveQueue($message, $reason, $this->queueOf($message)) !== null; + } + + /** + * A dead-letter publisher IS configured but the consumer's queue (topic) is unknown, so no `.dlq` can be + * named (a multi-topic Kafka subscription, a Pulsar client that reports no topic). ADR-0036 decision 3: the + * section 6.5 degrade applies, commit / acknowledge, drop and log, on Kafka as on Pulsar; the Dispatcher does + * NOT rethrow, because that would redeliver a record that can never be addressed. Without any publisher the + * earlier rule is unchanged (Kafka rethrows). + */ + private function isUnaddressable(ConsumedMessage $message): bool + { + return $this->deadLetters !== null && $message instanceof HasQueue && $this->queueOf($message) === null; + } + + /** + * The consumer's queue (its topic), when the message knows it. + */ + private function queueOf(ConsumedMessage $message): ?string + { + return $message instanceof HasQueue ? $message->queue() : null; + } + + /** + * Drop a message the Dispatcher cannot dead-letter: acknowledge it and log on Pulsar, and on Kafka when a + * publisher is set but the topic is unknown ({@see self::isUnaddressable()}); rethrow on Kafka without any + * publisher, where a normal return would commit the offset and lose the record, and for a message type the + * Dispatcher does not know (the previous behaviour). `$strategy` marks the `dead_letter` strategy + * degrading to `delete` (§3): that is the configured behaviour, so every non-Kafka type acks and logs. + */ + private function dropOrRethrow(ConsumedMessage $message, string $why, ?Throwable $e = null, bool $strategy = false): void + { + if ($message instanceof PulsarMessage || $this->isUnaddressable($message) || ($strategy && ! $message instanceof KafkaMessage)) { + $this->drop($message, $why); + + return; + } + + throw $e ?? new UnknownUrnException($why); + } + + /** + * Acknowledge-and-drop bookkeeping: tell `$onDrop`, or log. Never logs the body. + */ + private function drop(ConsumedMessage $message, string $why): void + { + if ($this->onDrop !== null) { + ($this->onDrop)($message, $why); + + return; + } + + $id = $message->getMeta()['id'] ?? null; + error_log(sprintf( + '[babelqueue] dropped a message (id %s, delivery %d): %s', + is_string($id) && $id !== '' ? $id : 'unknown', + $this->counted($message), + $why, + )); } } diff --git a/src/Contracts/HasMessageProperties.php b/src/Contracts/HasMessageProperties.php new file mode 100644 index 0000000..ef31695 --- /dev/null +++ b/src/Contracts/HasMessageProperties.php @@ -0,0 +1,22 @@ + + */ + public function properties(): array; +} diff --git a/src/Contracts/HasQueue.php b/src/Contracts/HasQueue.php new file mode 100644 index 0000000..1191006 --- /dev/null +++ b/src/Contracts/HasQueue.php @@ -0,0 +1,20 @@ +.dlq` for a message that carries no `meta.queue` of its own + * (a poison body, a delivery rejected by the version gate). Implemented by + * {@see \BabelQueue\Transport\PulsarMessage}. + */ +interface HasQueue +{ + /** + * The consumer's queue (the topic's local name, e.g. `orders` for + * `persistent://public/default/orders`), or null when the consumer was not told its topic. + */ + public function queue(): ?string; +} diff --git a/src/Transport/Internal/WebSocketConnection.php b/src/Transport/Internal/WebSocketConnection.php new file mode 100644 index 0000000..cfa5fb0 --- /dev/null +++ b/src/Transport/Internal/WebSocketConnection.php @@ -0,0 +1,252 @@ + $secure, + 'host' => $parts['host'], + 'port' => $parts['port'] ?? ($secure ? 443 : 80), + 'path' => rtrim($parts['path'] ?? '', '/'), + ]; + } + + /** + * Connect and upgrade. `$path` is the request target, query string included. + * + * @throws BabelQueueException when the connection or the upgrade fails + */ + public static function open(string $serviceUrl, string $path, ?string $authToken, float $timeoutSeconds): self + { + $service = self::parseService($serviceUrl); + if ($service === null) { + throw new \InvalidArgumentException(sprintf('Not a Pulsar WebSocket service URL: [%s].', $serviceUrl)); + } + + $address = sprintf('%s://%s:%d', $service['secure'] ? 'ssl' : 'tcp', $service['host'], $service['port']); + $stream = @stream_socket_client($address, $errno, $errstr, $timeoutSeconds, STREAM_CLIENT_CONNECT); + if ($stream === false) { + throw new BabelQueueException(sprintf('Cannot connect to the Pulsar WebSocket service %s: %s (%d)', $address, $errstr, $errno)); + } + + $seconds = (int) floor($timeoutSeconds); + stream_set_timeout($stream, $seconds, (int) (($timeoutSeconds - $seconds) * 1_000_000)); + $connection = new self($stream); + + try { + $key = base64_encode(random_bytes(16)); + $request = sprintf("GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: %s\r\nSec-WebSocket-Version: 13\r\n", $path, $service['host'], $service['port'], $key); + if ($authToken !== null && $authToken !== '') { + $request .= 'Authorization: Bearer ' . $authToken . "\r\n"; + } + $connection->write($request . "\r\n"); + + $head = ''; + while (! str_contains($head, "\r\n\r\n")) { + $byte = fread($stream, 1); // byte-wise: nothing past the header block is consumed + if ($byte === false || $byte === '') { + throw new BabelQueueException('The Pulsar WebSocket handshake was cut off: ' . trim($head)); + } + $head .= $byte; + if (strlen($head) > 16384) { + throw new BabelQueueException('The Pulsar WebSocket handshake response is too large.'); + } + } + + $status = strstr($head, "\r\n", true); + if ($status === false || ! str_contains($status, ' 101')) { + throw new BabelQueueException('The Pulsar WebSocket upgrade was refused: ' . ($status === false ? 'no status line' : $status)); + } + + $expected = base64_encode(sha1($key . self::GUID, true)); + if (preg_match('/^Sec-WebSocket-Accept:\s*(\S+)\s*$/im', $head, $m) !== 1 || ! hash_equals($expected, $m[1])) { + throw new BabelQueueException('The Pulsar WebSocket handshake returned a wrong Sec-WebSocket-Accept.'); + } + } catch (\Throwable $e) { + $connection->close(); + + throw $e; + } + + return $connection; + } + + public function sendText(string $data): void + { + $this->send(0x1, $data); + } + + /** Best effort: a close that fails changes nothing. */ + public function sendClose(): void + { + try { + $this->send(0x8, pack('n', 1000)); + } catch (BabelQueueException) { + } + } + + public function close(): void + { + if (is_resource($this->stream)) { + fclose($this->stream); + } + } + + /** + * Read one complete (possibly fragmented) data message; answers pings, fails on a close frame. + * + * @param float|null $waitSeconds null = wait for the connection's own timeout (an error); a number = give up + * quietly after that long when NOTHING has arrived and return null (a frame that has started is read whole) + * + * @throws BabelQueueException on a close, a timeout, or a broken connection + */ + public function readMessage(?float $waitSeconds = null): ?string + { + if ($waitSeconds !== null && ! $this->hasData($waitSeconds)) { + return null; + } + + $message = ''; + while (true) { + $head = $this->readExact(2); + $fin = (ord($head[0]) & 0x80) !== 0; + $opcode = ord($head[0]) & 0x0F; + $length = ord($head[1]) & 0x7F; + if ($length === 126) { + $length = self::unpackInt('n', $this->readExact(2)); + } elseif ($length === 127) { + $length = self::unpackInt('J', $this->readExact(8)); + } + if ($length > 16_777_216) { + throw new BabelQueueException('A Pulsar WebSocket message is unreasonably large.'); + } + $mask = (ord($head[1]) & 0x80) !== 0 ? $this->readExact(4) : null; + $data = $length > 0 ? $this->readExact($length) : ''; + if ($mask !== null && $length > 0) { + $data ^= str_pad($mask, $length, $mask); + } + + if ($opcode === 0x8) { + throw new BabelQueueException('The Pulsar WebSocket service closed the connection.'); + } + if ($opcode === 0x9) { + $this->send(0xA, $data); + + continue; + } + if ($opcode === 0xA) { + continue; + } + + $message .= $data; + if ($fin) { + return $message; + } + } + } + + /** True when a byte can be read now or within `$seconds`; bytes already buffered by PHP count. */ + private function hasData(float $seconds): bool + { + $meta = stream_get_meta_data($this->stream); + if ($meta['unread_bytes'] > 0) { + return true; + } + + $read = [$this->stream]; + $write = $except = null; + $whole = (int) floor($seconds); + $ready = @stream_select($read, $write, $except, $whole, (int) (($seconds - $whole) * 1_000_000)); + if ($ready === false) { + throw new BabelQueueException('Waiting on the Pulsar WebSocket connection failed.'); + } + + return $ready > 0; + } + + private function send(int $opcode, string $data): void + { + $length = strlen($data); + $header = chr(0x80 | ($opcode & 0x0F)); + if ($length < 126) { + $header .= chr(0x80 | $length); + } elseif ($length < 65536) { + $header .= chr(0x80 | 126) . pack('n', $length); + } else { + $header .= chr(0x80 | 127) . pack('J', $length); + } + + $mask = random_bytes(4); + $this->write($header . $mask . ($data ^ str_pad($mask, $length, $mask))); + } + + private function write(string $bytes): void + { + $total = strlen($bytes); + for ($written = 0; $written < $total;) { + $n = @fwrite($this->stream, substr($bytes, $written)); + if ($n === false || $n === 0) { + throw new BabelQueueException('Writing to the Pulsar WebSocket service failed.'); + } + $written += $n; + } + } + + private function readExact(int $length): string + { + $buffer = ''; + while (strlen($buffer) < $length) { + $chunk = fread($this->stream, max(1, $length - strlen($buffer))); + if ($chunk === false || $chunk === '') { + $meta = stream_get_meta_data($this->stream); + throw new BabelQueueException($meta['timed_out'] ? 'Timed out waiting on the Pulsar WebSocket service.' : 'The Pulsar WebSocket connection closed unexpectedly.'); + } + $buffer .= $chunk; + } + + return $buffer; + } + + private static function unpackInt(string $format, string $bytes): int + { + $unpacked = unpack($format, $bytes); + + return is_array($unpacked) ? (int) $unpacked[1] : 0; + } +} diff --git a/src/Transport/KafkaConsumer.php b/src/Transport/KafkaConsumer.php index f315cdd..23b2cb6 100644 --- a/src/Transport/KafkaConsumer.php +++ b/src/Transport/KafkaConsumer.php @@ -50,6 +50,11 @@ * opt-in: with no `$deadLetterProducer` the record degrades to commit-and-drop (§6.5), so it is never * redelivered in a loop. If the DLQ write itself throws, the offset is **not** committed and the * exception propagates (a later commit would otherwise silently skip the record). + * + * **The DLQ topic is the consumer's own topic** (ADR-0036 decision 3), never the body's `meta.queue` and never + * `default.dlq`: the frame's `topic`, else the `$topic` argument, else the client's single subscribed topic + * ({@see KafkaSubscriptionDescriptor}). When none of them names it (a multi-topic subscription is ambiguous), the + * record is committed and dropped with an `error_log` line (section 6.5), so a partition is never wedged. */ final class KafkaConsumer { @@ -64,11 +69,14 @@ final class KafkaConsumer * (a {@see KeyedKafkaProducer} also carries the record key); * null = commit-and-drop (§6.5, DLQ is opt-in) * @param string $dlqSuffix the dead-letter-topic suffix (`.dlq`) + * @param string|null $topic the topic this consumer reads, used when a record frame carries no `topic`; null = ask + * the client ({@see KafkaSubscriptionDescriptor}, single-topic subscriptions only) */ public function __construct( private readonly KafkaConsumerClient $client, private readonly ?KafkaProducer $deadLetterProducer = null, private readonly string $dlqSuffix = '.dlq', + private readonly ?string $topic = null, ) { } @@ -100,6 +108,7 @@ public function receive(): ?KafkaMessage $raw['payload'], self::UNSUPPORTED_SCHEMA_URN_PREFIX . $rejectedVersion, $raw['key'] ?? null, + $this->sourceTopic($raw), ); } @@ -121,7 +130,32 @@ public function receive(): ?KafkaMessage // Carry the raw bq- headers so the §6.4/§6.5 retry-topic machinery can recover a retry // record's original work topic (bq-original-topic) across hops. - return new KafkaMessage($envelope, $raw['headers'], $raw['payload'], null, $raw['key'] ?? null); + return new KafkaMessage($envelope, $raw['headers'], $raw['payload'], null, $raw['key'] ?? null, $this->sourceTopic($raw)); + } + + /** + * The topic a record was read from: the frame's `topic`, else the configured `$topic`, else the client's single + * subscribed topic. Null when none of them says (a multi-topic subscription is ambiguous). + * + * @param array{payload: string, headers: array, key?: string|null, topic?: string|null} $raw + */ + private function sourceTopic(array $raw): ?string + { + $topic = $raw['topic'] ?? null; + if (is_string($topic) && $topic !== '') { + return $topic; + } + if ($this->topic !== null && $this->topic !== '') { + return $this->topic; + } + if ($this->client instanceof KafkaSubscriptionDescriptor) { + $topics = array_values(array_filter($this->client->subscribedTopics(), static fn (string $t): bool => $t !== '')); + if (count($topics) === 1) { + return $topics[0]; + } + } + + return null; } /** @@ -133,9 +167,15 @@ public function receive(): ?KafkaMessage private function quarantine(array $raw): void { if ($this->deadLetterProducer !== null) { - $topic = $raw['topic'] ?? null; - if (! is_string($topic) || $topic === '') { - $topic = PoisonBody::queueHint($raw['payload']) ?? 'default'; + $topic = $this->sourceTopic($raw); + if ($topic === null) { + // ADR-0036 decision 3: the DLQ is `.dlq`, never the body's `meta.queue`, never + // `default.dlq`. With no queue to name it this is the "no publisher" case of section 6.5: commit and + // drop, loudly. Throwing instead would leave the offset uncommitted and wedge the partition. + error_log('[babelqueue] A poison Kafka record was dropped: the topic it was read from is unknown (no "topic" in the receive() frame, no $topic, no single-topic KafkaSubscriptionDescriptor), so no ' . $this->dlqSuffix . ' can be named.'); + $this->commit(); + + return; } $headerAttempts = $raw['headers']['bq-attempts'] ?? null; diff --git a/src/Transport/KafkaConsumerClient.php b/src/Transport/KafkaConsumerClient.php index 3066de8..c96fc73 100644 --- a/src/Transport/KafkaConsumerClient.php +++ b/src/Transport/KafkaConsumerClient.php @@ -26,9 +26,12 @@ interface KafkaConsumerClient * empty string is a real, empty key) — carried through the retry / dead-letter writes * ({@see KeyedKafkaProducer}). * - * The optional `topic` is the topic the record was read from (omit it, or null, when unknown); a - * poison record is dead-lettered to `.dlq`, falling back to the body's `meta.queue` - * (default `default`) when it is absent. + * The `topic` is the topic the record was read from. Report it: it is the consumer's queue, and it alone + * names `.dlq` for both the {@see \BabelQueue\Consume\Dispatcher} and a poison record + * (ADR-0036 decision 3). Without it the consumer falls back to its `$topic` argument, then to a single-topic + * {@see KafkaSubscriptionDescriptor}. If none names it, the body's `meta.queue` is still never used and + * `default.dlq` is never written: a poison record is committed and dropped with a log line (section 6.5), and the + * Dispatcher rethrows for a message that needs the DLQ. * * @return array{payload: string, headers: array, key?: string|null, topic?: string|null}|null */ diff --git a/src/Transport/KafkaMessage.php b/src/Transport/KafkaMessage.php index bf3095f..9d64002 100644 --- a/src/Transport/KafkaMessage.php +++ b/src/Transport/KafkaMessage.php @@ -7,6 +7,7 @@ use BabelQueue\Codec\EnvelopeCodec; use BabelQueue\Contracts\ConsumedMessage; use BabelQueue\Contracts\HasHeaders; +use BabelQueue\Contracts\HasQueue; use BabelQueue\Contracts\HasRawBody; use BabelQueue\Contracts\HasRecordKey; @@ -28,7 +29,7 @@ * URN is {@see KafkaConsumer::UNSUPPORTED_SCHEMA_URN_PREFIX} plus the rejected value and the envelope * is only `['raw' => , 'attempts' => n]`. */ -final class KafkaMessage implements ConsumedMessage, HasHeaders, HasRawBody, HasRecordKey +final class KafkaMessage implements ConsumedMessage, HasHeaders, HasQueue, HasRawBody, HasRecordKey { /** * @param array $envelope the decoded envelope, with `attempts` already reconciled @@ -36,6 +37,8 @@ final class KafkaMessage implements ConsumedMessage, HasHeaders, HasRawBody, Has * @param string|null $rawBody the record value as received (kept for poison dead-lettering) * @param string|null $routedUrn the URN to route on instead of the body's (the version-gate sentinel), or null * @param string|null $key the record key bytes as received (null = null key; '' = a real, empty key) + * @param string|null $topic the topic the record was read from (the consumer's own queue), when the client + * reported it; names `.dlq` for the {@see \BabelQueue\Consume\Dispatcher} */ public function __construct( private readonly array $envelope, @@ -43,6 +46,7 @@ public function __construct( private readonly ?string $rawBody = null, private readonly ?string $routedUrn = null, private readonly ?string $key = null, + private readonly ?string $topic = null, ) { } @@ -123,6 +127,15 @@ public function recordKey(): ?string return $this->key; } + /** + * The topic this record was read from (a Kafka topic is the BabelQueue queue), or null when the consumer + * client did not report it. It, not the envelope's `meta.queue`, picks `.dlq` (ADR-0036 decision 3). + */ + public function queue(): ?string + { + return $this->topic !== null && $this->topic !== '' ? $this->topic : null; + } + /** * The raw record value as received, or null when it was not captured. */ diff --git a/src/Transport/KafkaSubscriptionDescriptor.php b/src/Transport/KafkaSubscriptionDescriptor.php new file mode 100644 index 0000000..55b1a9a --- /dev/null +++ b/src/Transport/KafkaSubscriptionDescriptor.php @@ -0,0 +1,21 @@ + + */ + public function subscribedTopics(): array; +} diff --git a/src/Transport/PulsarConsumer.php b/src/Transport/PulsarConsumer.php index 7ef622f..fa8ce07 100644 --- a/src/Transport/PulsarConsumer.php +++ b/src/Transport/PulsarConsumer.php @@ -5,6 +5,9 @@ namespace BabelQueue\Transport; use BabelQueue\Codec\EnvelopeCodec; +use BabelQueue\Codec\StrictBodyClassifier; +use BabelQueue\Consume\DeadLetterWriteFailed; +use BabelQueue\Exceptions\BabelQueueException; use Closure; use Throwable; @@ -25,8 +28,8 @@ * returns the trimmed, non-blank property and falls back to the body's URN * ({@see EnvelopeCodec::urn()}) when it is absent or blank. The body is always decoded and * {@see PulsarMessage::envelope()} is always the body's envelope. A body without a `job` is never - * routed, even when `bq-job` is set (it yields an empty URN, which the dispatcher treats as an - * unknown URN). + * routed, even when `bq-job` is set (it yields an empty URN, which the + * {@see \BabelQueue\Consume\Dispatcher} treats as a poison body, ADR-0036 decision 4). * * Per §5.2/§5.7 the **`attempts` counter** is the authoritative `bq-attempts` property (valid only * when it is plain decimal digits, `^[0-9]+$`, no greater than 2147483647), falling back to the @@ -51,6 +54,47 @@ * the rejected version. The envelope of such a message is only `['raw' => , 'attempts' => n]`, * so a dead-letter copy keeps the producer's bytes in the same `{"raw": ...}` shape as a poison body. * + * **Queue name (ADR-0036 decision 3).** A Pulsar message frame carries no topic, so the consumer is told + * the queue it reads: pass `$queue` (the topic's local name, e.g. `orders` for + * `persistent://public/default/orders`), or let the client adapter add an optional `topic` key to the + * frame it returns from `receive()` (the full topic or the local name; the part after the last `/` is + * used). The name is exposed as {@see PulsarMessage::queue()} and is where the + * {@see \BabelQueue\Consume\DeadLetterPublisher} writes `.dlq` for a message with no `meta.queue` + * (a poison body, a version-gated delivery); without it the {@see \BabelQueue\Consume\Dispatcher} cannot name the + * queue and treats such a message like "no publisher" (nothing is written to `default.dlq`). The queue is + * **derived** when not given: from a client that implements {@see PulsarConsumerDescriptor} (its `topic()`), else + * from the frame's `topic` key. + * + * **Out of the box.** The SDK's own {@see PulsarWebSocketSubscription} is the consumer client for a Pulsar WebSocket + * service; it reports its topic, subscription type and service URL, so `new PulsarConsumer($subscription)` needs no + * further argument and the retry cap holds on every subscription type. + * + * **Retry is a re-publish (ADR-0036 decision 1, the Go model).** A negative acknowledgement advances Pulsar's + * redelivery count only on a `Shared` / `Key_Shared` subscription; on `Exclusive` / `Failover` it does not, so a + * nack-only retry would never reach the {@see \BabelQueue\Consume\Dispatcher}'s `maxAttempts`. So, through + * the {@see PulsarWebSocketClient} producer seam, {@see self::retry()} — which {@see self::consume()} calls + * when a handler throws — re-publishes the message to the **same topic** with `bq-attempts` + 1 (and the body's + * top-level `attempts` kept in sync for an envelope; a poison or version-gated body keeps its exact bytes and + * properties), then acknowledges the original, so the bound holds whatever the subscription type. A failed + * re-publish leaves the original unacknowledged (negative ack). + * + * **The consumer derives what it needs** (an explicit argument always wins; the API is unchanged): + * - the retry producer: `$retryProducer`, else the client itself when it also implements + * {@see PulsarWebSocketClient} (one connection object that reads and publishes), else a + * {@see PulsarWebSocketProducer} on the Pulsar WebSocket service (`$serviceUrl`, else the client's + * {@see PulsarServiceEndpoint::serviceUrl()}): the producer endpoint is the consumer's own host, and the tenant, + * namespace and topic come from the consumer's real topic; + * - the topic: `$queue` (built into `persistent:////`), else a client that implements + * {@see PulsarConsumerDescriptor} (its `topic()`, used as-is when it is a full name), else the frame's `topic`; + * - the subscription type: `$subscriptionType`, else {@see PulsarConsumerDescriptor::subscriptionType()}. + * **Fail-closed, only when nothing can be derived.** A nack-only retry is correct on `Shared` / `Key_Shared` only, + * and Pulsar cannot be asked which type a subscription has over this seam, so an unknown type is treated as + * `Exclusive` (Pulsar's own default). When no producer can be derived at all (no `$retryProducer`, no `$serviceUrl`, + * and a client that is neither a {@see PulsarWebSocketClient} nor a {@see PulsarServiceEndpoint}) the constructor throws unless the subscription is known to be `Shared` / + * `Key_Shared` (a clear setup error instead of a retry that can never reach the cap), and a retry whose topic is + * still unknown throws too. To keep a nack-only consumer, say so: pass `subscriptionType: 'Shared'` (or + * `'Key_Shared'`), or implement {@see PulsarConsumerDescriptor}. + * * URN dispatch, `on_unknown_urn` and DLQ routing can be handled by the framework-less * {@see \BabelQueue\Consume\Dispatcher} runtime (or a framework adapter): it is callable, so * `consume($dispatcher, $shouldStop)` dispatches on {@see PulsarMessage::getUrn()} and @@ -69,16 +113,143 @@ final class PulsarConsumer private readonly ?Closure $onRoutingMismatch; + /** The retry producer: given, else the client itself when it can publish. */ + private readonly ?PulsarWebSocketClient $producer; + + /** The full topic the client reads, when it said so; the retry re-publishes to it as-is. */ + private readonly ?string $retryTopic; + + /** The queue (local topic name) the client reported, used when `$queue` is not given. */ + private readonly ?string $clientQueue; + + /** `Exclusive` | `Failover` | `Shared` | `Key_Shared`, or null when unknown. */ + private readonly ?string $subscriptionType; + /** * @param (callable(string, string, array): mixed)|null $onRoutingMismatch * optional diagnostic hook `fn($propertyUrn, $bodyUrn, $envelope)` called when the * `bq-job` property and the body URN differ; default no-op, its errors are swallowed + * @param string|null $queue the queue this consumer reads (its topic's local name); names `.dlq` + * for a message without `meta.queue`. Null = take it from the frame's optional `topic` key, else unknown + * @param PulsarWebSocketClient|null $retryProducer the producer {@see self::retry()} re-publishes through; + * null = the client itself when it is a {@see PulsarWebSocketClient}, else a nack (`Shared` / `Key_Shared` only) + * @param string|null $tenant the tenant, used with `$namespace` to build the re-publish topic from a LOCAL topic + * name only; never guessed (`public` is not assumed): a local name without both is a retry error + * @param string|null $namespace the namespace, see `$tenant` + * @param string|null $subscriptionType `Exclusive` | `Failover` | `Shared` | `Key_Shared` (case-insensitive); + * null = ask the client ({@see PulsarConsumerDescriptor}), else unknown, which counts as `Exclusive` + * + * @param string|null $serviceUrl the Pulsar WebSocket service (`ws://host:8080`, `http://host:8080`, ...) the retry + * producer is built on ({@see PulsarWebSocketProducer}); null = ask the client ({@see PulsarServiceEndpoint}) + * + * @param string|null $serviceAuthToken the bearer token of the derived producer; null = the client's + * {@see PulsarServiceEndpoint::authToken()}. Needed on a cluster that requires authentication when `$serviceUrl` is given + * @param bool $topicFromFrame declare that the adapter puts the FULL topic name in every frame's `topic` key and that + * the re-publish topic may be taken from there. Without it (and without a descriptor topic, or `$queue` with + * `$tenant` and `$namespace`) the constructor refuses a producer that would not know where to publish + * + * @throws \InvalidArgumentException When `$subscriptionType` is not a Pulsar subscription type, or the service URL is not one + * @throws BabelQueueException When no retry producer can be derived and the subscription is not known to be + * `Shared` / `Key_Shared` (`Exclusive`, `Failover` or unknown) */ public function __construct( private readonly PulsarWebSocketConsumerClient $client, ?callable $onRoutingMismatch = null, + private readonly ?string $queue = null, + ?PulsarWebSocketClient $retryProducer = null, + private readonly ?string $tenant = null, + private readonly ?string $namespace = null, + ?string $subscriptionType = null, + ?string $serviceUrl = null, + ?string $serviceAuthToken = null, + bool $topicFromFrame = false, ) { $this->onRoutingMismatch = $onRoutingMismatch === null ? null : Closure::fromCallable($onRoutingMismatch); + + $descriptor = $client instanceof PulsarConsumerDescriptor ? $client : null; + + $this->producer = $retryProducer + ?? ($client instanceof PulsarWebSocketClient ? $client : null) + ?? self::producerFromServiceUrl($serviceUrl, $serviceAuthToken, $client); + + $topic = $descriptor?->topic(); + $topic = $topic !== null && $topic !== '' ? $topic : null; + $this->retryTopic = $topic !== null && str_contains($topic, '://') ? $topic : null; + $this->clientQueue = $topic !== null ? self::localName($topic) : null; + + $type = $subscriptionType ?? $descriptor?->subscriptionType(); + $this->subscriptionType = $type === null ? null : self::normaliseSubscriptionType($type); + + if ($this->producer === null && $this->nackCannotBoundRetry()) { + throw new BabelQueueException(sprintf( + 'A %s Pulsar subscription needs a retry producer: a negative acknowledgement does not advance the redelivery count there, so the retry could never reach maxAttempts, and none could be derived (no $retryProducer, no $serviceUrl, and the client implements neither %s nor %s). Pass one of them, or declare a Shared / Key_Shared subscription (subscriptionType: or %s) if that is what it is.', + $this->subscriptionType ?? 'unknown (treated as Exclusive)', + PulsarWebSocketClient::class, + PulsarServiceEndpoint::class, + PulsarConsumerDescriptor::class, + )); + } + + if ( + $this->producer !== null + && $this->nackCannotBoundRetry() + && ! $topicFromFrame + && $this->retryTopic === null + && ! ($queue !== null && $tenant !== null && $namespace !== null) + ) { + throw new BabelQueueException(sprintf( + 'The retry of an %s Pulsar subscription re-publishes to the topic the message was read from, and this consumer cannot know it: the client reports no full topic (%s::topic()) and no $tenant and $namespace were given with $queue. Report a full topic, pass them, or pass topicFromFrame: true if every frame carries the full "topic".', + $this->subscriptionType ?? 'unknown (treated as Exclusive)', + PulsarConsumerDescriptor::class, + )); + } + } + + /** + * The retry producer built on the Pulsar WebSocket service: the explicit `$serviceUrl`, else the client's own + * ({@see PulsarServiceEndpoint}). Null when neither names one. Its tenant, namespace and topic come from the + * topic of each message at retry time, never from here. + */ + private static function producerFromServiceUrl(?string $serviceUrl, ?string $serviceAuthToken, PulsarWebSocketConsumerClient $client): ?PulsarWebSocketClient + { + $token = $serviceAuthToken; + if ($client instanceof PulsarServiceEndpoint) { + if ($serviceUrl === null || $serviceUrl === '') { + $serviceUrl = $client->serviceUrl(); + } + $token ??= $client->authToken(); + } + + return $serviceUrl !== null && $serviceUrl !== '' ? new PulsarWebSocketProducer($serviceUrl, $token) : null; + } + + /** + * True unless the subscription is known to be `Shared` / `Key_Shared`: on `Exclusive` / `Failover` a nack leaves + * the native redelivery count unchanged, and an unknown type is treated as `Exclusive` (fail-closed). + */ + private function nackCannotBoundRetry(): bool + { + return $this->subscriptionType !== 'Shared' && $this->subscriptionType !== 'Key_Shared'; + } + + private static function normaliseSubscriptionType(string $type): string + { + foreach (['Exclusive', 'Failover', 'Shared', 'Key_Shared'] as $known) { + if (strcasecmp($type, $known) === 0) { + return $known; + } + } + + throw new \InvalidArgumentException(sprintf('Unknown Pulsar subscription type [%s]; expected Exclusive, Failover, Shared or Key_Shared.', $type)); + } + + /** The part after the last `/` of a topic name (the whole name when it has none), null when empty. */ + private static function localName(string $topic): ?string + { + $slash = strrpos($topic, '/'); + $name = $slash === false ? $topic : substr($topic, $slash + 1); + + return $name !== '' ? $name : null; } /** @@ -94,6 +265,8 @@ public function receive(): ?PulsarMessage } $properties = $raw['properties']; + $queue = $this->queueOf($raw); + $topic = $this->topicOf($raw); // §5.7 step 3: gate on bq-schema-version before the body is decoded. $rejectedVersion = self::unsupportedSchemaVersion($properties['bq-schema-version'] ?? null); @@ -105,6 +278,9 @@ public function receive(): ?PulsarMessage $raw['messageId'], $raw['payload'], self::UNSUPPORTED_SCHEMA_URN_PREFIX . $rejectedVersion, + $queue, + $properties, + $topic, ); } @@ -116,6 +292,12 @@ public function receive(): ?PulsarMessage if ($attempts === null) { $body = $envelope['attempts'] ?? 0; $attempts = is_int($body) ? $body : 0; + + // A poison body has no decodable `attempts` (pulsar_exhaustion: its input is `bq-attempts` and the + // redelivery count only), even when the lenient codec could read one (e.g. a wrong-typed `trace_id`). + if ($attempts > 0 && StrictBodyClassifier::isPoison($raw['payload'])) { + $attempts = 0; + } } $envelope['attempts'] = max($attempts, $raw['redeliveryCount']); @@ -134,7 +316,42 @@ public function receive(): ?PulsarMessage } } - return new PulsarMessage($envelope, $raw['messageId'], $raw['payload'], $routedUrn); + return new PulsarMessage($envelope, $raw['messageId'], $raw['payload'], $routedUrn, $queue, $properties, $topic); + } + + /** + * The queue of a frame: the configured one, else the local name of the frame's optional `topic` + * (the part after the last `/`), else null. + * + * @param array $raw + */ + private function topicOf(array $raw): ?string + { + if ($this->retryTopic !== null) { + return $this->retryTopic; + } + + $topic = $raw['topic'] ?? null; + + return is_string($topic) && str_contains($topic, '://') ? $topic : null; + } + + /** + * @param array $raw + */ + private function queueOf(array $raw): ?string + { + if ($this->queue !== null && $this->queue !== '') { + return $this->queue; + } + + if ($this->clientQueue !== null) { + return $this->clientQueue; + } + + $topic = $raw['topic'] ?? null; + + return is_string($topic) ? self::localName($topic) : null; } /** @@ -207,9 +424,85 @@ public function release(PulsarMessage $message): void $this->client->negativeAcknowledge($message->messageId()); } + /** + * Retry a message: re-publish it to the same topic with `bq-attempts` + 1 and acknowledge the original + * (see the class docblock). The original is acknowledged only after the re-publish succeeded; if the + * re-publish fails it is negatively acknowledged and the failure is logged, so nothing is lost. With no + * producer (possible only on a known `Shared` / `Key_Shared` subscription, see the class docblock) it is + * {@see self::release()}. + */ + public function retry(PulsarMessage $message): void + { + if ($this->producer === null) { + $this->release($message); // Shared / Key_Shared (or an unknown type): the nack advances the count + + return; + } + + // The topic the message was read from, whole (descriptor / frame): never rebuilt from a guessed tenant and + // namespace. A local name is completed only with an explicit $tenant and $namespace. + $queue = $message->queue(); + $topic = $message->topic(); + if ($topic === null && $queue !== null && $this->tenant !== null && $this->namespace !== null) { + $topic = sprintf('persistent://%s/%s/%s', $this->tenant, $this->namespace, $queue); + } + + if ($topic === null) { + if ($this->nackCannotBoundRetry()) { + throw new BabelQueueException('Cannot retry: the full topic of this Pulsar consumer is unknown, and the copy must not go to a guessed tenant/namespace. Let the client implement ' . PulsarConsumerDescriptor::class . ' with a full topic name, put a full "topic" in the frame, or pass $tenant and $namespace with $queue.'); + } + + $this->release($message); + + return; + } + + $next = $message->attempts() + 1; + $properties = $message->properties(); + $properties['bq-attempts'] = (string) $next; + + try { + $this->producer->publish($topic, $this->retryBody($message, $next), $properties); + } catch (Throwable $e) { + error_log('[babelqueue] Pulsar retry re-publish failed, the message is negatively acknowledged: ' . $e->getMessage()); + $this->release($message); + + return; + } + + $this->acknowledge($message); + } + + /** + * The body of the re-published copy: a decodable envelope is re-encoded with its top-level `attempts` set to + * `$next` (in sync with `bq-attempts`); a poison or version-gated body keeps its exact bytes (the count lives + * in the property, which is authoritative). + */ + private function retryBody(PulsarMessage $message, int $next): string + { + $raw = $message->rawBody() ?? ''; + + if ( + str_starts_with($message->getUrn(), self::UNSUPPORTED_SCHEMA_URN_PREFIX) + || $message->getUrn() === '' + || StrictBodyClassifier::isPoison($raw) + ) { + return $raw; + } + + try { + $envelope = $message->envelope(); + $envelope['attempts'] = $next; + + return EnvelopeCodec::encode($envelope); + } catch (Throwable) { + return $raw; + } + } + /** * Run the consume loop: receive each message and pass it to $handler; acknowledge on success, - * release (redeliver) if the handler throws. Idle (null) receives are skipped. The loop runs + * retry (see {@see self::retry()}) if the handler throws. Idle (null) receives are skipped. The loop runs * until $shouldStop() returns true (omit it to run forever, the standard worker model). * * @param callable(PulsarMessage): void $handler @@ -227,10 +520,14 @@ public function consume(callable $handler, ?callable $shouldStop = null): void try { $handler($message); $this->acknowledge($message); + } catch (DeadLetterWriteFailed) { + // The dead-letter write failed (section 5.5): leave the original unacknowledged and nack it, so it is + // redelivered and the write is tried again; a retry copy would move it on without a DLQ record. + $this->release($message); } catch (Throwable) { - // At-least-once: redeliver on failure. A handler owns poison handling via + // At-least-once: retry on failure. A handler owns poison handling via // $message->attempts() (e.g. ack-and-drop or republish to .dlq past a cap). - $this->release($message); + $this->retry($message); } } } diff --git a/src/Transport/PulsarConsumerDescriptor.php b/src/Transport/PulsarConsumerDescriptor.php new file mode 100644 index 0000000..3d125fb --- /dev/null +++ b/src/Transport/PulsarConsumerDescriptor.php @@ -0,0 +1,31 @@ +.dlq`), the topic of the retry re-publish and whether a negative acknowledgement can bound the retry + * at all (ADR-0036 decision 1) without extra configuration. An adapter that also implements + * {@see PulsarWebSocketClient} (it owns the connection, so it can publish too) is used as the retry producer. + * + * Both values are known to the adapter: it derives the WebSocket endpoint + * (`ws:///ws/v2/consumer/persistent////`) from them. + */ +interface PulsarConsumerDescriptor +{ + /** + * The topic this subscription reads: the full name (`persistent://public/default/orders`) or its local name + * (`orders`; the consumer then builds the full name from its tenant and namespace). An empty string means + * the adapter does not know it. + */ + public function topic(): string; + + /** + * The subscription type: `Exclusive`, `Failover`, `Shared` or `Key_Shared` (case-insensitive), or null when + * the adapter does not know it. + */ + public function subscriptionType(): ?string; +} diff --git a/src/Transport/PulsarMessage.php b/src/Transport/PulsarMessage.php index 71934e2..f7792af 100644 --- a/src/Transport/PulsarMessage.php +++ b/src/Transport/PulsarMessage.php @@ -6,6 +6,8 @@ use BabelQueue\Codec\EnvelopeCodec; use BabelQueue\Contracts\ConsumedMessage; +use BabelQueue\Contracts\HasMessageProperties; +use BabelQueue\Contracts\HasQueue; use BabelQueue\Contracts\HasRawBody; /** @@ -18,18 +20,25 @@ * is never decoded: the routing URN is {@see PulsarConsumer::UNSUPPORTED_SCHEMA_URN_PREFIX} plus the * rejected value and the envelope is only `['raw' => , 'attempts' => n]`. */ -final class PulsarMessage implements ConsumedMessage, HasRawBody +final class PulsarMessage implements ConsumedMessage, HasMessageProperties, HasQueue, HasRawBody { /** * @param array $envelope the decoded envelope, with `attempts` already reconciled * @param string|null $rawBody the body bytes as received (kept for poison dead-lettering) * @param string|null $routedUrn the `bq-job` property URN to route on, or null to use the body's + * @param string|null $queue the consumer's queue (its topic's local name), or null when unknown; names `.dlq` + * @param array $properties the message properties as received (kept so a retry re-publish carries them) + * @param string|null $topic the full topic name (`persistent://tenant/namespace/topic`) the consumer reads, when + * known; the retry re-publishes to exactly this topic */ public function __construct( private readonly array $envelope, private readonly string $messageId, private readonly ?string $rawBody = null, private readonly ?string $routedUrn = null, + private readonly ?string $queue = null, + private readonly array $properties = [], + private readonly ?string $topic = null, ) { } @@ -89,6 +98,32 @@ public function envelope(): array return $this->envelope; } + /** + * The consumer's queue (the topic's local name), or null when the consumer was not told its topic. + */ + public function queue(): ?string + { + return $this->queue; + } + + /** + * The full topic name this message was read from, when the consumer knows it, else null. + */ + public function topic(): ?string + { + return $this->topic; + } + + /** + * The message properties as received (`bq-job`, `bq-attempts`, ...), string to string. + * + * @return array + */ + public function properties(): array + { + return $this->properties; + } + /** * The raw body as received, or null when it was not captured. */ diff --git a/src/Transport/PulsarServiceEndpoint.php b/src/Transport/PulsarServiceEndpoint.php new file mode 100644 index 0000000..a50d3a0 --- /dev/null +++ b/src/Transport/PulsarServiceEndpoint.php @@ -0,0 +1,29 @@ +/ws/v2/consumer/persistent////`; the producer endpoint is the + * same service, `ws:///ws/v2/producer/persistent///`, and its tenant, namespace and + * topic come from the consumer's real (full) topic, never from a guess. + */ +interface PulsarServiceEndpoint +{ + /** + * The base URL of the Pulsar WebSocket service: `ws://host:8080`, `wss://host:8443`, `http://host:8080` or + * `https://host:8443` (`http`/`https` mean `ws`/`wss`), with an optional path prefix when a proxy mounts the + * service below one. An empty string means the adapter does not know it. + */ + public function serviceUrl(): string; + + /** A bearer token for the producer connection (the `Authorization: Bearer` header), or null for none. */ + public function authToken(): ?string; +} diff --git a/src/Transport/PulsarTransport.php b/src/Transport/PulsarTransport.php index b18acad..dfeaea2 100644 --- a/src/Transport/PulsarTransport.php +++ b/src/Transport/PulsarTransport.php @@ -5,7 +5,7 @@ namespace BabelQueue\Transport; use BabelQueue\Codec\EnvelopeCodec; -use BabelQueue\Contracts\Transport; +use BabelQueue\Contracts\HeaderPublisher; /** * A framework-less **Apache Pulsar** producer — PHP's path to the §5 binding via Pulsar's native @@ -33,7 +33,7 @@ * {@see Transport} is publish-only, like the Redis/AMQP/SQS/STOMP/Kafka transports); PHP consumes * Pulsar via a framework worker. */ -final class PulsarTransport implements Transport +final class PulsarTransport implements HeaderPublisher { public function __construct( private readonly PulsarWebSocketClient $client, @@ -44,6 +44,16 @@ public function __construct( } public function publish(string $payload, ?string $queue = null): ?string + { + return $this->publishWithHeaders($payload, [], $queue); + } + + /** + * Publish with extra message properties ({@see HeaderPublisher}): they are merged over the `bq-` projection, so a + * caller can carry the incoming route (`bq-job`) onto a copy (the dead-letter record). No headers is exactly + * {@see self::publish()}. + */ + public function publishWithHeaders(string $payload, array $headers, ?string $queue = null): ?string { $target = $queue ?? $this->defaultQueue; // Decoded only to project the transport headers; the payload is published verbatim, so a @@ -51,7 +61,7 @@ public function publish(string $payload, ?string $queue = null): ?string $envelope = EnvelopeCodec::decode($payload, static function (): void {}); $topic = sprintf('persistent://%s/%s/%s', $this->tenant, $this->namespace, $target); - $this->client->publish($topic, $payload, $this->properties($envelope)); + $this->client->publish($topic, $payload, array_filter($headers, static fn (string $v, string $k): bool => $k !== '' && $v !== '', ARRAY_FILTER_USE_BOTH) + $this->properties($envelope)); $meta = is_array($envelope['meta'] ?? null) ? $envelope['meta'] : []; $id = $meta['id'] ?? null; diff --git a/src/Transport/PulsarWebSocketConsumerClient.php b/src/Transport/PulsarWebSocketConsumerClient.php index 4760162..763c30d 100644 --- a/src/Transport/PulsarWebSocketConsumerClient.php +++ b/src/Transport/PulsarWebSocketConsumerClient.php @@ -24,7 +24,11 @@ interface PulsarWebSocketConsumerClient * Receive the next message from the subscription, or null if none arrived within the client's * read window (so the caller can poll without blocking forever). * - * @return array{messageId: string, payload: string, properties: array, redeliveryCount: int}|null + * The optional `topic` is the topic this subscription reads (the full `persistent://...` name or its + * local name); {@see PulsarConsumer} uses it as the queue for `.dlq` when it was not given one + * explicitly (ADR-0036 decision 3). A message frame itself carries no topic, so the adapter adds it. + * + * @return array{messageId: string, payload: string, properties: array, redeliveryCount: int, topic?: string|null}|null */ public function receive(): ?array; diff --git a/src/Transport/PulsarWebSocketProducer.php b/src/Transport/PulsarWebSocketProducer.php new file mode 100644 index 0000000..fe54625 --- /dev/null +++ b/src/Transport/PulsarWebSocketProducer.php @@ -0,0 +1,87 @@ +/ws/v2/producer/persistent///`, sends one JSON frame (`payload` base64, + * `properties`), waits for the broker's `{"result":"ok"}` and closes. A non-`ok` result, a refused connection, a + * failed handshake or a timeout throws {@see BabelQueueException}, so a failed publish is never silently lost (GR-1). + * + * It exists so {@see PulsarConsumer} can re-publish a retry on its own (derived from a client's + * {@see PulsarServiceEndpoint}). It speaks only what the producer endpoint needs (RFC 6455 over + * `stream_socket_client`, `ssl://` for `wss`/`https` with the peer verified), so GR-7 holds: no extension and no + * library. The tenant, namespace and topic are parsed from the FULL topic name passed to `publish()`; a local name + * has no tenant and is refused. It never sets `eventTime` (the body's `meta.created_at` is authoritative). + */ +final class PulsarWebSocketProducer implements PulsarWebSocketClient +{ + /** + * @param string $serviceUrl `ws://host:8080`, `wss://...`, `http://...` or `https://...` (optional path prefix) + * @param string|null $authToken sent as `Authorization: Bearer ` when set + * @param float $timeoutSeconds connect and read timeout + */ + public function __construct( + private readonly string $serviceUrl, + private readonly ?string $authToken = null, + private readonly float $timeoutSeconds = 10.0, + ) { + if (WebSocketConnection::parseService($serviceUrl) === null) { + throw new \InvalidArgumentException(sprintf('Not a Pulsar WebSocket service URL: [%s] (expected ws://host:port, wss://, http:// or https://).', $serviceUrl)); + } + } + + public function publish(string $topic, string $payload, array $properties): void + { + $path = $this->producerPath($topic); + $frame = json_encode([ + 'payload' => base64_encode($payload), + 'properties' => $properties === [] ? new \stdClass() : $properties, + 'context' => bin2hex(random_bytes(8)), + ], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); + + $connection = WebSocketConnection::open($this->serviceUrl, $path, $this->authToken, $this->timeoutSeconds); + + try { + $connection->sendText($frame); + $reply = json_decode((string) $connection->readMessage(), true); + + if (! is_array($reply) || ($reply['result'] ?? null) !== 'ok') { + $detail = 'unreadable reply'; + if (is_array($reply)) { + $reason = $reply['errorMsg'] ?? $reply['result'] ?? null; + $detail = is_string($reason) ? $reason : 'unknown'; + } + throw new BabelQueueException(sprintf('Pulsar producer rejected the message for [%s]: %s', $topic, $detail)); + } + + $connection->sendClose(); // the broker already acknowledged the message + } finally { + $connection->close(); + } + } + + /** The producer endpoint path for a full topic: `/ws/v2/producer/persistent///`. */ + public function producerPath(string $topic): string + { + $parts = PulsarWebSocketSubscription::splitTopic($topic); + if ($parts === null) { + throw new BabelQueueException(sprintf('Cannot derive the Pulsar producer endpoint: [%s] is not a full topic name (persistent:////).', $topic)); + } + + $service = WebSocketConnection::parseService($this->serviceUrl); + + return sprintf('%s/ws/v2/producer/%s', $service['path'] ?? '', self::topicPath($parts)); + } + + /** @param array{domain: string, tenant: string, namespace: string, topic: string} $parts */ + public static function topicPath(array $parts): string + { + return sprintf('%s/%s/%s/%s', $parts['domain'], rawurlencode($parts['tenant']), rawurlencode($parts['namespace']), rawurlencode($parts['topic'])); + } +} diff --git a/src/Transport/PulsarWebSocketSubscription.php b/src/Transport/PulsarWebSocketSubscription.php new file mode 100644 index 0000000..6d397c5 --- /dev/null +++ b/src/Transport/PulsarWebSocketSubscription.php @@ -0,0 +1,217 @@ +/ws/v2/consumer/persistent////` and implements the + * {@see PulsarWebSocketConsumerClient} seam {@see PulsarConsumer} consumes through. It also implements + * {@see PulsarConsumerDescriptor} (full topic, subscription type) and {@see PulsarServiceEndpoint} (service URL and + * token), so `new PulsarConsumer(new PulsarWebSocketSubscription(...))` needs nothing more: the queue, the + * re-publish topic and the subscription type are known, and the retry producer is derived on the same service, so + * the `maxAttempts` cap holds on an `Exclusive` / `Failover` subscription too (ADR-0036 decision 1). + * + * Pure userland (RFC 6455 over `stream_socket_client`, `ssl://` for `wss` / `https`), so GR-7 holds. The connection + * is opened lazily by the first {@see self::receive()}; when it breaks, `receive()` throws + * {@see BabelQueueException} and the next call opens a new one (messages that were delivered but not acknowledged + * on the old connection are redelivered by the broker). `receive()` returns null when nothing arrived within the + * read window, so a loop can poll and stop. + */ +final class PulsarWebSocketSubscription implements PulsarWebSocketConsumerClient, PulsarConsumerDescriptor, PulsarServiceEndpoint +{ + private const SUBSCRIPTION_TYPES = ['Exclusive', 'Failover', 'Shared', 'Key_Shared']; + + private ?WebSocketConnection $connection = null; + + private readonly string $subscriptionType; + + /** + * @param string $serviceUrl `ws://host:8080`, `wss://...`, `http://...` or `https://...` (optional path prefix) + * @param string $topic the FULL topic name, `persistent:////` + * @param string $subscription the subscription name + * @param string $subscriptionType `Exclusive` (Pulsar's default), `Failover`, `Shared` or `Key_Shared`: what the + * subscription is (it is sent to the broker, which creates it so) + * @param string|null $authToken sent as `Authorization: Bearer ` (consumer and derived producer) + * @param float $readWindowSeconds how long `receive()` waits for a message before returning null + * @param float $timeoutSeconds connect and in-frame read timeout + * @param int|null $receiverQueueSize Pulsar's consumer prefetch (null = the broker default) + * + * @throws \InvalidArgumentException When the URL, the topic or the subscription type is not valid + */ + public function __construct( + private readonly string $serviceUrl, + private readonly string $topic, + private readonly string $subscription, + string $subscriptionType = 'Exclusive', + private readonly ?string $authToken = null, + private readonly float $readWindowSeconds = 1.0, + private readonly float $timeoutSeconds = 10.0, + private readonly ?int $receiverQueueSize = null, + ) { + if (WebSocketConnection::parseService($serviceUrl) === null) { + throw new \InvalidArgumentException(sprintf('Not a Pulsar WebSocket service URL: [%s] (expected ws://host:port, wss://, http:// or https://).', $serviceUrl)); + } + if (self::splitTopic($topic) === null) { + throw new \InvalidArgumentException(sprintf('[%s] is not a full topic name (persistent:////).', $topic)); + } + if ($subscription === '') { + throw new \InvalidArgumentException('A Pulsar subscription needs a name.'); + } + + $normalised = null; + foreach (self::SUBSCRIPTION_TYPES as $known) { + if (strcasecmp($subscriptionType, $known) === 0) { + $normalised = $known; + } + } + if ($normalised === null) { + throw new \InvalidArgumentException(sprintf('Unknown Pulsar subscription type [%s]; expected Exclusive, Failover, Shared or Key_Shared.', $subscriptionType)); + } + $this->subscriptionType = $normalised; + } + + /** + * @return array{domain: string, tenant: string, namespace: string, topic: string}|null null when it is not a full name + */ + public static function splitTopic(string $topic): ?array + { + if (preg_match('#^(persistent|non-persistent)://([^/]+)/([^/]+)/(.+)$#', $topic, $m) !== 1) { + return null; + } + + return ['domain' => $m[1], 'tenant' => $m[2], 'namespace' => $m[3], 'topic' => $m[4]]; + } + + public function receive(): ?array + { + $connection = $this->connection(); + + try { + $text = $connection->readMessage($this->readWindowSeconds); + } catch (BabelQueueException $e) { + $this->close(); + + throw $e; + } + + if ($text === null) { + return null; + } + + $frame = json_decode($text, true); + if (! is_array($frame) || ! is_string($frame['messageId'] ?? null) || ! is_string($frame['payload'] ?? null)) { + return null; // a control / error frame, not a delivery + } + + $payload = base64_decode($frame['payload'], true); + $properties = []; + if (is_array($frame['properties'] ?? null)) { + foreach ($frame['properties'] as $key => $value) { + if (is_string($value)) { + $properties[(string) $key] = $value; + } + } + } + $redelivery = $frame['redeliveryCount'] ?? 0; + + return [ + 'messageId' => $frame['messageId'], + 'payload' => $payload === false ? '' : $payload, + 'properties' => $properties, + 'redeliveryCount' => is_int($redelivery) ? $redelivery : 0, + 'topic' => $this->topic, + ]; + } + + public function acknowledge(string $messageId): void + { + $this->sendControl(['messageId' => $messageId]); + } + + public function negativeAcknowledge(string $messageId): void + { + $this->sendControl(['type' => 'negativeAcknowledge', 'messageId' => $messageId]); + } + + public function topic(): string + { + return $this->topic; + } + + public function subscriptionType(): string + { + return $this->subscriptionType; + } + + public function serviceUrl(): string + { + return $this->serviceUrl; + } + + public function authToken(): ?string + { + return $this->authToken; + } + + /** Close the connection (a later `receive()` opens a new one). */ + public function close(): void + { + if ($this->connection !== null) { + $this->connection->sendClose(); + $this->connection->close(); + $this->connection = null; + } + } + + public function __destruct() + { + $this->close(); + } + + /** The consumer endpoint path with its query string. */ + public function consumerPath(): string + { + $parts = self::splitTopic($this->topic); + $service = WebSocketConnection::parseService($this->serviceUrl); + assert($parts !== null && $service !== null); + + $query = ['subscriptionType' => $this->subscriptionType]; + if ($this->receiverQueueSize !== null) { + $query['receiverQueueSize'] = (string) $this->receiverQueueSize; + } + + return sprintf( + '%s/ws/v2/consumer/%s/%s?%s', + $service['path'], + PulsarWebSocketProducer::topicPath($parts), + rawurlencode($this->subscription), + http_build_query($query), + ); + } + + private function connection(): WebSocketConnection + { + return $this->connection ??= WebSocketConnection::open($this->serviceUrl, $this->consumerPath(), $this->authToken, $this->timeoutSeconds); + } + + /** @param array $frame */ + private function sendControl(array $frame): void + { + if ($this->connection === null) { + throw new BabelQueueException('The Pulsar consumer connection is closed; the message will be redelivered by the broker.'); + } + + try { + $this->connection->sendText(json_encode($frame, JSON_THROW_ON_ERROR)); + } catch (BabelQueueException $e) { + $this->close(); + + throw $e; + } + } +} diff --git a/tests/Consume/DeadLetterPublisherTest.php b/tests/Consume/DeadLetterPublisherTest.php index 83d1b3b..9705c33 100644 --- a/tests/Consume/DeadLetterPublisherTest.php +++ b/tests/Consume/DeadLetterPublisherTest.php @@ -72,15 +72,39 @@ public function test_publishes_an_annotated_envelope_to_the_dlq(): void $this->assertSame('php', $dl['lang']); } - public function test_falls_back_to_the_default_queue_when_meta_queue_is_missing(): void + public function test_uses_the_callers_queue_when_meta_queue_is_missing(): void { $transport = Mockery::mock(Transport::class); - $transport->shouldReceive('publish')->once()->with(Mockery::any(), 'default.dlq')->andReturn(null); + $transport->shouldReceive('publish')->once()->with(Mockery::any(), 'invoices.dlq')->andReturn(null); - (new DeadLetterPublisher($transport))->publish($this->message(['lang' => 'php']), 'unknown_urn', null); + (new DeadLetterPublisher($transport))->publish($this->message(['lang' => 'php']), 'unknown_urn', null, null, 'invoices'); $this->addToAssertionCount(1); } + public function test_never_writes_default_dlq_when_no_queue_can_be_named(): void + { + $transport = Mockery::mock(Transport::class); + $transport->shouldNotReceive('publish'); + $publisher = new DeadLetterPublisher($transport); + $message = $this->message(['lang' => 'php']); + + $this->assertNull($publisher->resolveQueue($message, 'unknown_urn')); + $this->assertNull($publisher->resolveQueue($message, 'unknown_urn', '')); + + $this->expectException(\BabelQueue\Exceptions\BabelQueueException::class); + $publisher->publish($message, 'unknown_urn', null); + } + + public function test_a_poison_copy_takes_the_callers_queue_not_the_bodys_meta_queue(): void + { + $raw = '{"job":"urn:babel:orders:created","trace_id":"t","data":[1],"meta":{"id":"m","queue":"orders","schema_version":1},"attempts":0}'; + $message = new PulsarMessage(EnvelopeCodec::decode($raw, static function (): void {}), 'mid-1', $raw); + $publisher = new DeadLetterPublisher(Mockery::mock(Transport::class)); + + $this->assertSame('topic-queue', $publisher->resolveQueue($message, 'poison', 'topic-queue')); + $this->assertNull($publisher->resolveQueue($message, 'poison'), 'a poison copy has no queue of its own'); + } + public function test_honours_a_custom_suffix(): void { $transport = Mockery::mock(Transport::class); @@ -92,7 +116,7 @@ public function test_honours_a_custom_suffix(): void } /** @return array|null the decoded DLQ body published to $dlq */ - private function publishAndCapture(ConsumedMessage $message, string $dlq = 'default.dlq'): ?array + private function publishAndCapture(ConsumedMessage $message, string $dlq = 'orders.dlq'): ?array { $captured = null; $transport = Mockery::mock(Transport::class); @@ -106,7 +130,7 @@ private function publishAndCapture(ConsumedMessage $message, string $dlq = 'defa $dlq, )->andReturn(null); - (new DeadLetterPublisher($transport))->publish($message, 'poison', null); + (new DeadLetterPublisher($transport))->publish($message, 'poison', null, null, 'orders'); return $captured; } diff --git a/tests/Consume/DispatcherTerminationTest.php b/tests/Consume/DispatcherTerminationTest.php new file mode 100644 index 0000000..b5e4850 --- /dev/null +++ b/tests/Consume/DispatcherTerminationTest.php @@ -0,0 +1,641 @@ + payload, queue */ + private array $published = []; + + /** @var list */ + private array $dropped = []; + + protected function setUp(): void + { + EnvelopeCodec::setWarningHandler(static function (): void { + }); + } + + protected function tearDown(): void + { + EnvelopeCodec::setWarningHandler(null); + } + + private function publisher(): DeadLetterPublisher + { + $transport = Mockery::mock(Transport::class); + $transport->shouldReceive('publish')->andReturnUsing(function (string $payload, ?string $queue): ?string { + $this->published[] = [$payload, (string) $queue]; + + return null; + }); + + return new DeadLetterPublisher($transport); + } + + private function onDrop(): \Closure + { + return function (ConsumedMessage $m, string $why): void { + $this->dropped[] = $why; + }; + } + + /** + * A Pulsar delivery through the real consumer. + * + * @param array $properties + * @param array $frame extra frame keys (e.g. `topic`) + */ + private function pulsar(string $body = self::BODY, int $redelivery = 0, array $properties = [], ?string $queue = 'orders', array $frame = []): PulsarMessage + { + $client = Mockery::mock(PulsarWebSocketConsumerClient::class); + $client->shouldReceive('receive')->once()->andReturn( + ['messageId' => 'mid-1', 'payload' => $body, 'properties' => $properties, 'redeliveryCount' => $redelivery] + $frame, + ); + $message = (new PulsarConsumer($client, null, $queue, subscriptionType: 'Shared'))->receive(); + $this->assertNotNull($message); + + return $message; + } + + private function kafka(string $body = self::BODY, int $attempts = 0, ?string $topic = 'orders'): KafkaMessage + { + $envelope = EnvelopeCodec::decode($body); + $envelope['attempts'] = $attempts; + + return new KafkaMessage($envelope, [], $body, null, null, $topic); + } + + private function failing(Dispatcher $dispatcher): Dispatcher + { + return $dispatcher->on(self::URN, static function (): void { + throw new RuntimeException('boom'); + }); + } + + /** @return array */ + private function record(int $index = 0): array + { + return json_decode($this->published[$index][0], true, 512, JSON_THROW_ON_ERROR); + } + + // -- poison route --------------------------------------------------------------------------------- + + public function test_a_pulsar_poison_body_goes_to_the_consumers_queue_dlq_not_default(): void + { + $dispatcher = (new Dispatcher(deadLetters: $this->publisher())) + ->on(self::URN, static fn () => self::fail('a poison body must never reach a handler')); + + $dispatcher($this->pulsar('{not json')); + + $this->assertCount(1, $this->published); + $this->assertSame('orders.dlq', $this->published[0][1]); + $record = $this->record(); + $this->assertSame('{not json', $record['raw']); + $this->assertSame('poison', $record['dead_letter']['reason']); + $this->assertSame('orders', $record['dead_letter']['original_queue']); + } + + public function test_the_queue_comes_from_the_frames_topic_when_the_consumer_was_not_given_one(): void + { + $dispatcher = new Dispatcher(deadLetters: $this->publisher()); + + $dispatcher($this->pulsar('{}', 0, [], null, ['topic' => 'persistent://public/default/invoices'])); + + $this->assertSame('invoices.dlq', $this->published[0][1]); + $this->assertSame('invoices', $this->record()['dead_letter']['original_queue']); + } + + public function test_an_explicit_consumer_queue_wins_over_the_frames_topic(): void + { + $dispatcher = new Dispatcher(deadLetters: $this->publisher()); + + $dispatcher($this->pulsar('{}', 0, [], 'orders', ['topic' => 'persistent://public/default/invoices'])); + + $this->assertSame('orders.dlq', $this->published[0][1]); + } + + /** A valid envelope with no `meta.queue` (a foreign producer), through a consumer that was not told its queue. */ + private const NO_META_QUEUE = '{"job":"urn:babel:orders:created","trace_id":"t-1","data":{},"attempts":0}'; + + public function test_default_dlq_is_never_written_an_unnameable_queue_is_handled_like_no_publisher_on_pulsar(): void + { + // Pulsar, queue unknown: a poison body under `dead_letter` is acknowledged, dropped and logged. + (new Dispatcher('dead_letter', 3, $this->publisher(), onDrop: $this->onDrop()))($this->pulsar('{not json', 0, [], null)); + // ... an unknown URN under `dead_letter` likewise (the envelope has no meta.queue either). + (new Dispatcher('dead_letter', 3, $this->publisher(), onDrop: $this->onDrop()))($this->pulsar(self::NO_META_QUEUE, 0, [], null)); + // ... a poison body under `fail` is retried, then dropped at the cap (the bounded path of "no publisher"). + $fail = new Dispatcher('fail', 3, $this->publisher(), onDrop: $this->onDrop()); + try { + $fail($this->pulsar('{not json', 0, [], null)); + $this->fail('delivery 1 is retried'); + } catch (UnknownUrnException) { + $this->addToAssertionCount(1); + } + $fail($this->pulsar('{not json', 2, [], null)); + // ... and a failing handler at the cap. + $this->failing(new Dispatcher('fail', 3, $this->publisher(), onDrop: $this->onDrop()))($this->pulsar(self::NO_META_QUEUE, 2, [], null)); + + $this->assertSame([], $this->published, 'nothing is written to default.dlq'); + $this->assertCount(4, $this->dropped); + } + + public function test_an_unnameable_queue_on_kafka_with_a_publisher_drops_and_logs_and_never_publishes(): void + { + // ADR-0036 decision 3: a publisher is set but the topic is unknown: section 6.5 degrade (commit, drop, log). + // A rethrow would redeliver a record that can never be addressed and wedge the partition. + $poison = new KafkaMessage(['attempts' => 0], [], '{not json'); + $noMeta = new KafkaMessage(EnvelopeCodec::decode(self::NO_META_QUEUE), [], self::NO_META_QUEUE); + + foreach ([$poison, $noMeta] as $message) { + (new Dispatcher('dead_letter', 3, $this->publisher(), onDrop: $this->onDrop()))($message); + } + + $this->assertSame([], $this->published, 'never default.dlq, never .dlq'); + $this->assertCount(2, $this->dropped); + } + + public function test_an_unnameable_queue_on_kafka_without_any_publisher_still_rethrows(): void + { + $poison = new KafkaMessage(['attempts' => 0], [], '{not json'); + $noMeta = new KafkaMessage(EnvelopeCodec::decode(self::NO_META_QUEUE), [], self::NO_META_QUEUE); + + foreach ([$poison, $noMeta] as $message) { + try { + (new Dispatcher('dead_letter', 3, onDrop: $this->onDrop()))($message); + $this->fail('without a publisher Kafka rethrows'); + } catch (UnknownUrnException) { + $this->addToAssertionCount(1); + } + } + + $this->assertSame([], $this->dropped); + } + + public function test_the_drop_is_for_pulsar_only_an_unknown_message_type_keeps_rethrowing(): void + { + $message = new class implements ConsumedMessage { + public function getUrn(): string + { + return DispatcherTerminationTest::URN; + } + + public function getTraceId(): string + { + return ''; + } + + public function getData(): array + { + return []; + } + + public function getMeta(): array + { + return ['queue' => 'orders']; + } + + public function attempts(): int + { + return 2; // counted 3 = the cap + } + + public function envelope(): array + { + return ['job' => DispatcherTerminationTest::URN, 'meta' => ['queue' => 'orders'], 'attempts' => 2]; + } + }; + + try { + $this->failing(new Dispatcher(onDrop: $this->onDrop()))($message); + $this->fail('a message type the Dispatcher does not know is never dropped on the cap'); + } catch (RuntimeException) { + $this->assertSame([], $this->dropped); + } + + // ... but the dead_letter strategy keeps degrading to delete (acked, now logged) as configured. + (new Dispatcher('dead_letter', onDrop: $this->onDrop()))(new class implements ConsumedMessage { + public function getUrn(): string + { + return 'urn:x'; + } + + public function getTraceId(): string + { + return ''; + } + + public function getData(): array + { + return []; + } + + public function getMeta(): array + { + return []; + } + + public function attempts(): int + { + return 0; + } + + public function envelope(): array + { + return ['job' => 'urn:x']; + } + }); + $this->assertCount(1, $this->dropped); + } + + public function test_a_poison_body_counts_only_bq_attempts_and_the_redelivery_count_not_a_body_attempts(): void + { + // Strictly poison (wrong-typed trace_id) yet decodable: the body says attempts 7. + $body = '{"job":"urn:babel:orders:created","trace_id":5,"attempts":7}'; + $message = $this->pulsar($body); + + $this->assertSame(0, $message->attempts()); + + try { + (new Dispatcher('fail', 3, onDrop: $this->onDrop()))($message); + $this->fail('delivery 1 (counted 1 < 3) is retried, not dropped'); + } catch (UnknownUrnException) { + $this->assertSame([], $this->dropped); + } + + // bq-attempts and the redelivery count still count. + $this->assertSame(2, $this->pulsar($body, 0, ['bq-attempts' => '2'])->attempts()); + $this->assertSame(4, $this->pulsar($body, 4)->attempts()); + } + + public function test_an_identity_less_object_and_a_wrong_typed_member_are_poison_with_their_bytes_kept(): void + { + $dispatcher = new Dispatcher(deadLetters: $this->publisher()); + + $wrongTyped = str_replace('"attempts":0', '"attempts":"3"', self::BODY); + $dispatcher($this->pulsar('{"trace_id":"t","data":{}}')); + $dispatcher($this->pulsar($wrongTyped)); + + $this->assertCount(2, $this->published); + $this->assertSame('poison', $this->record(0)['dead_letter']['reason']); + $this->assertSame('t', $this->record(0)['trace_id']); + $this->assertSame('poison', $this->record(1)['dead_letter']['reason']); + $this->assertStringContainsString('"attempts":"3"', $this->published[1][0], 'the original bytes are spliced, not re-encoded'); + } + + public function test_a_poison_body_is_not_retried_when_a_publisher_is_set(): void + { + $dispatcher = new Dispatcher('release', 5, $this->publisher()); + + $dispatcher($this->pulsar('{not json', 0)); // returns (ack), no throw, on its first delivery + + $this->assertCount(1, $this->published); + } + + public function test_a_poison_body_without_a_publisher_takes_the_bounded_fail_path(): void + { + foreach (['fail', 'release'] as $strategy) { + $dispatcher = new Dispatcher($strategy, 3, onDrop: $this->onDrop()); + + foreach ([0, 1] as $redelivery) { + try { + $dispatcher($this->pulsar('{not json', $redelivery)); + $this->fail('delivery ' . ($redelivery + 1) . ' must be retried'); + } catch (UnknownUrnException) { + $this->addToAssertionCount(1); + } + } + + $dispatcher($this->pulsar('{not json', 2)); // 3rd delivery: acknowledged and dropped + } + + $this->assertCount(2, $this->dropped); + $this->assertSame([], $this->published); + } + + public function test_poison_with_delete_drops_and_logs(): void + { + (new Dispatcher('delete', 3, $this->publisher(), onDrop: $this->onDrop()))($this->pulsar('{not json')); + + $this->assertSame([], $this->published); + $this->assertCount(1, $this->dropped); + } + + public function test_poison_with_dead_letter_and_no_publisher_drops_and_logs_on_pulsar_but_rethrows_on_kafka(): void + { + $dispatcher = new Dispatcher('dead_letter', onDrop: $this->onDrop()); + + $dispatcher($this->pulsar('{not json')); + $this->assertCount(1, $this->dropped); + + $this->expectException(UnknownUrnException::class); + $dispatcher(new KafkaMessage(['attempts' => 0], [], '{not json')); + } + + // -- cap, counted dead_letter.attempts ------------------------------------------------------------ + + public function test_the_default_cap_is_three_and_the_dlq_record_carries_the_counted_attempts(): void + { + $dispatcher = $this->failing(new Dispatcher(deadLetters: $this->publisher())); + + foreach ([0, 1] as $redelivery) { + try { + $dispatcher($this->pulsar(self::BODY, $redelivery)); + $this->fail('delivery ' . ($redelivery + 1) . ' must be rethrown'); + } catch (RuntimeException) { + $this->assertSame([], $this->published); + } + } + + $dispatcher($this->pulsar(self::BODY, 2)); // 3rd delivery + + $this->assertCount(1, $this->published); + $this->assertSame('orders.dlq', $this->published[0][1]); + $this->assertSame('failed', $this->record()['dead_letter']['reason']); + $this->assertSame(3, $this->record()['dead_letter']['attempts']); + } + + public function test_without_a_publisher_the_third_delivery_is_acknowledged_dropped_and_not_rethrown(): void + { + $dispatcher = $this->failing(new Dispatcher(onDrop: $this->onDrop())); + + $dispatcher($this->pulsar(self::BODY, 2)); // would throw before: no publisher -> unbounded redelivery + + $this->assertCount(1, $this->dropped); + $this->assertSame([], $this->published); + } + + public function test_the_default_drop_is_logged_through_error_log_without_the_body(): void + { + $file = tempnam(sys_get_temp_dir(), 'bq-log'); + $previous = ini_set('error_log', (string) $file); + + try { + $this->failing(new Dispatcher())($this->pulsar(self::BODY, 2)); + } finally { + ini_set('error_log', (string) $previous); + } + + $log = (string) file_get_contents((string) $file); + unlink((string) $file); + $this->assertStringContainsString('[babelqueue] dropped a message (id m-1, delivery 3)', $log); + $this->assertStringNotContainsString('order_id', $log); + } + + public function test_unknown_urn_fail_and_release_are_bounded_by_the_cap(): void + { + foreach (['fail', 'release'] as $strategy) { + $this->published = []; + $dispatcher = new Dispatcher($strategy, 3, $this->publisher()); + + try { + $dispatcher($this->pulsar(self::BODY, 1)); + $this->fail('delivery 2 must be retried'); + } catch (UnknownUrnException) { + $this->assertSame([], $this->published); + } + + $dispatcher($this->pulsar(self::BODY, 2)); + + $this->assertCount(1, $this->published, $strategy); + $this->assertSame('unknown_urn', $this->record()['dead_letter']['reason']); + $this->assertSame(3, $this->record()['dead_letter']['attempts']); + } + } + + public function test_a_direct_dead_letter_keeps_the_delivered_attempts(): void + { + (new Dispatcher('dead_letter', 3, $this->publisher()))($this->pulsar(self::BODY, 1)); + + $this->assertSame('unknown_urn', $this->record()['dead_letter']['reason']); + $this->assertSame(1, $this->record()['dead_letter']['attempts'], 'unchanged: informational, not counted'); + } + + public function test_max_attempts_zero_means_no_cap(): void + { + $dispatcher = $this->failing(new Dispatcher('fail', 0, $this->publisher(), onDrop: $this->onDrop())); + + $this->expectException(RuntimeException::class); + try { + $dispatcher($this->pulsar(self::BODY, 500)); + } finally { + $this->assertSame([], $this->published); + $this->assertSame([], $this->dropped); + } + } + + public function test_unknown_urn_dead_letter_without_a_publisher_logs_and_acks_on_pulsar(): void + { + (new Dispatcher('dead_letter', onDrop: $this->onDrop()))($this->pulsar()); + + $this->assertCount(1, $this->dropped); + } + + public function test_unknown_urn_delete_stays_a_silent_ack_and_drop(): void + { + (new Dispatcher('delete', onDrop: $this->onDrop()))($this->pulsar()); + + $this->assertSame([], $this->dropped); + $this->assertSame([], $this->published); + } + + public function test_a_version_gated_pulsar_delivery_is_dead_lettered_to_the_consumers_queue(): void + { + $dispatcher = new Dispatcher('dead_letter', 3, $this->publisher()); + + $dispatcher($this->pulsar(self::BODY, 0, ['bq-schema-version' => '2'])); + + $this->assertSame('orders.dlq', $this->published[0][1]); + $this->assertSame('unknown_urn', $this->record()['dead_letter']['reason']); + } + + // -- Kafka: the Dispatcher never drops ------------------------------------------------------------ + + public function test_on_kafka_a_failing_handler_at_the_cap_without_a_publisher_rethrows_and_never_drops(): void + { + $dispatcher = $this->failing(new Dispatcher(onDrop: $this->onDrop())); + + try { + $dispatcher($this->kafka(self::BODY, 2)); + $this->fail('the exception must be rethrown so the retry router or the consumer decides'); + } catch (RuntimeException) { + $this->assertSame([], $this->dropped); + $this->assertSame([], $this->published); + } + } + + public function test_on_kafka_unknown_urn_fail_and_release_at_the_cap_without_a_publisher_rethrow(): void + { + foreach (['fail', 'release', 'dead_letter'] as $strategy) { + try { + (new Dispatcher($strategy, 3, onDrop: $this->onDrop()))($this->kafka(self::BODY, 2)); + $this->fail($strategy . ' must rethrow on Kafka'); + } catch (UnknownUrnException) { + $this->assertSame([], $this->dropped); + } + } + } + + public function test_on_kafka_the_cap_with_a_publisher_dead_letters_to_the_topic_dlq_with_counted_attempts(): void + { + $dispatcher = $this->failing(new Dispatcher(deadLetters: $this->publisher())); + + $dispatcher($this->kafka(self::BODY, 2)); + + $this->assertSame('orders.dlq', $this->published[0][1]); + $this->assertSame('failed', $this->record()['dead_letter']['reason']); + $this->assertSame(3, $this->record()['dead_letter']['attempts']); + } + + public function test_on_kafka_the_dlq_is_named_by_the_consumers_topic_never_by_meta_queue(): void + { + // meta.queue says "orders" (or "default"), the record was read from "billing": billing.dlq, only. + foreach ([self::BODY, str_replace('"queue":"orders"', '"queue":"default"', self::BODY)] as $body) { + $this->published = []; + $this->failing(new Dispatcher(deadLetters: $this->publisher()))($this->kafka($body, 2, 'billing')); + + $this->assertSame(['billing.dlq'], array_column($this->published, 1)); + $this->assertSame($this->record()['meta']['queue'], $this->record()['dead_letter']['original_queue']); + } + } + + public function test_on_kafka_poison_is_dead_lettered_to_the_topic_dlq(): void + { + $dispatcher = new Dispatcher('dead_letter', 3, $this->publisher()); + $dispatcher(new KafkaMessage(['attempts' => 0], [], '{not json', null, null, 'billing')); + + $this->assertSame(['billing.dlq'], array_column($this->published, 1)); + $this->assertSame('poison', $this->record()['dead_letter']['reason']); + } + + public function test_on_kafka_an_unknown_topic_with_a_publisher_drops_and_logs_even_with_meta_queue(): void + { + // A failing handler at the cap, an unknown-URN fail at the cap, both with a publisher and no topic. + foreach ([null, ''] as $topic) { + $this->failing(new Dispatcher(deadLetters: $this->publisher(), onDrop: $this->onDrop()))($this->kafka(self::BODY, 2, $topic)); + (new Dispatcher('fail', 3, $this->publisher(), onDrop: $this->onDrop()))($this->kafka(self::BODY, 2, $topic)); + } + + $this->assertSame([], $this->published, 'meta.queue must not name the DLQ'); + $this->assertCount(4, $this->dropped, 'committed, dropped and logged: never a rethrow that wedges the partition'); + } + + public function test_on_kafka_an_unknown_topic_below_the_cap_is_still_a_normal_retry(): void + { + $this->expectException(\RuntimeException::class); + + $this->failing(new Dispatcher(deadLetters: $this->publisher(), onDrop: $this->onDrop()))($this->kafka(self::BODY, 0, null)); + } + + public function test_on_kafka_an_unknown_topic_without_a_publisher_still_rethrows_at_the_cap(): void + { + $this->expectException(\RuntimeException::class); + + $this->failing(new Dispatcher(onDrop: $this->onDrop()))($this->kafka(self::BODY, 2, null)); + } + + public function test_kafka_message_exposes_the_source_topic_as_its_queue(): void + { + $this->assertSame('billing', (new KafkaMessage(['attempts' => 0], [], '{}', null, null, 'billing'))->queue()); + $this->assertNull((new KafkaMessage(['attempts' => 0], [], '{}'))->queue()); + $this->assertNull((new KafkaMessage(['attempts' => 0], [], '{}', null, null, ''))->queue()); + } + + public function test_on_kafka_below_the_cap_the_failure_is_rethrown_as_before(): void + { + $dispatcher = $this->failing(new Dispatcher(deadLetters: $this->publisher())); + + $this->expectException(RuntimeException::class); + $dispatcher($this->kafka(self::BODY, 1)); + } + + public function test_max_attempts_zero_leaves_the_cap_to_the_kafka_retry_router(): void + { + $dispatcher = $this->failing(new Dispatcher('fail', 0, $this->publisher())); + + try { + $dispatcher($this->kafka(self::BODY, 9)); + $this->fail('rethrown'); + } catch (Throwable) { + $this->assertSame([], $this->published); + } + } + + // -- DeadLetterPublisher -------------------------------------------------------------------------- + + public function test_the_target_is_the_callers_queue_and_meta_queue_only_fills_original_queue(): void + { + $publisher = $this->publisher(); + $withMeta = $this->pulsar(self::BODY); + $noMeta = new PulsarMessage(['job' => self::URN, 'attempts' => 1], 'mid-2'); + + $publisher->publish($withMeta, 'failed', null, 5, 'other'); + $publisher->publish($noMeta, 'failed', null, null, 'other'); + $publisher->publish($withMeta, 'failed', null, 5); // no caller queue: the message's own meta.queue + + $this->assertSame(['other.dlq', 'other.dlq', 'orders.dlq'], array_column($this->published, 1)); + $this->assertSame('orders', $this->record(0)['dead_letter']['original_queue']); + $this->assertSame('other', $this->record(1)['dead_letter']['original_queue']); + $this->assertSame(5, $this->record(0)['dead_letter']['attempts']); + $this->assertSame(1, $this->record(1)['dead_letter']['attempts'], 'no explicit attempts: the delivered value'); + } + + public function test_a_producers_meta_queue_never_picks_the_dlq_on_pulsar(): void + { + // EnvelopeCodec::make() defaults meta.queue to "default"; a producer may also have used any other name. + foreach (['default', 'billing'] as $metaQueue) { + $this->published = []; + $body = str_replace('"queue":"orders"', '"queue":"' . $metaQueue . '"', self::BODY); + $dispatcher = $this->failing(new Dispatcher(deadLetters: $this->publisher())); + + $dispatcher($this->pulsar($body, 2)); // consumer queue `orders`, third delivery + + $this->assertSame('orders.dlq', $this->published[0][1], $metaQueue); + $this->assertSame($metaQueue, $this->record()['dead_letter']['original_queue'], $metaQueue); + } + + // ... and the unknown-URN strategy likewise. + $this->published = []; + (new Dispatcher('dead_letter', 3, $this->publisher()))($this->pulsar(str_replace('"queue":"orders"', '"queue":"default"', str_replace(self::URN, 'urn:babel:other', self::BODY)))); + $this->assertSame('orders.dlq', $this->published[0][1]); + } + + public function test_a_pulsar_consumer_with_an_unknown_queue_does_not_fall_back_to_the_producers_meta_queue(): void + { + $dispatcher = $this->failing(new Dispatcher(deadLetters: $this->publisher(), onDrop: $this->onDrop())); + + $dispatcher($this->pulsar(self::BODY, 2, [], null)); // meta.queue is orders, but the consumer's queue is unknown + + $this->assertSame([], $this->published); + $this->assertCount(1, $this->dropped); + } +} diff --git a/tests/Consume/DispatcherTest.php b/tests/Consume/DispatcherTest.php index 6aa0d7d..84513bb 100644 --- a/tests/Consume/DispatcherTest.php +++ b/tests/Consume/DispatcherTest.php @@ -63,6 +63,7 @@ public function test_on_retry_hook_receives_the_backoff_delay_and_the_failure_is $calls = []; $boom = new RuntimeException('boom'); $dispatcher = (new Dispatcher( + maxAttempts: 0, // no cap: attempts 2 would otherwise be exhausted (default 3) and dropped retryBackoff: new \BabelQueue\Consume\RetryBackoff(retryBackoff: 2, multiplier: 3), onRetry: function (ConsumedMessage $m, int $delay, \Throwable $e) use (&$calls): void { $calls[] = [$m->attempts(), $delay, $e->getMessage()]; @@ -175,9 +176,15 @@ public function test_unknown_urn_dead_letter_publishes_then_acks(): void public function test_unknown_urn_dead_letter_degrades_to_delete_without_a_publisher(): void { - // dead_letter strategy but no publisher → degrade to delete (no throw, nothing published). - (new Dispatcher('dead_letter'))($this->message('urn:babel:x')); - $this->addToAssertionCount(1); + // dead_letter strategy but no publisher → degrade to delete (no throw, nothing published), visibly: + // the drop is reported to the onDrop hook (ADR-0036 decision 3). + $dropped = []; + (new Dispatcher('dead_letter', onDrop: function (ConsumedMessage $m, string $why) use (&$dropped): void { + $dropped[] = $m->getUrn() . ' | ' . $why; + }))($this->message('urn:babel:x')); + + $this->assertCount(1, $dropped); + $this->assertStringContainsString('urn:babel:x', $dropped[0]); } public function test_plugs_into_a_consumer_consume_loop_and_acks(): void @@ -199,7 +206,7 @@ public function test_plugs_into_a_consumer_consume_loop_and_acks(): void }); $calls = 0; - (new PulsarConsumer($client))->consume($dispatcher, function () use (&$calls): bool { + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume($dispatcher, function () use (&$calls): bool { return $calls++ >= 1; }); diff --git a/tests/IdentityVerdictConformanceTest.php b/tests/IdentityVerdictConformanceTest.php new file mode 100644 index 0000000..7e1b412 --- /dev/null +++ b/tests/IdentityVerdictConformanceTest.php @@ -0,0 +1,28 @@ +assertSame($case['expect'], $verdict, (string) $case['name']); + }); + } +} diff --git a/tests/KafkaConsumerTest.php b/tests/KafkaConsumerTest.php index d11e577..f4d2b43 100644 --- a/tests/KafkaConsumerTest.php +++ b/tests/KafkaConsumerTest.php @@ -12,6 +12,7 @@ use BabelQueue\Transport\KafkaConsumerClient; use BabelQueue\Transport\KafkaMessage; use BabelQueue\Transport\KafkaProducer; +use BabelQueue\Transport\KafkaSubscriptionDescriptor; use BabelQueue\Transport\KeyedKafkaProducer; use Mockery; use Mockery\Adapter\Phpunit\MockeryPHPUnitIntegration; @@ -35,11 +36,11 @@ final class KafkaConsumerTest extends TestCase /** * @param array $headers - * @return array{payload: string, headers: array} + * @return array{payload: string, headers: array, topic?: string} */ - private function raw(array $headers = [], string $payload = self::ENVELOPE): array + private function raw(array $headers = [], string $payload = self::ENVELOPE, ?string $topic = 'orders'): array { - return ['payload' => $payload, 'headers' => $headers]; + return ['payload' => $payload, 'headers' => $headers] + ($topic === null ? [] : ['topic' => $topic]); } public function test_receive_decodes_the_envelope_and_exposes_the_inbound_view(): void @@ -274,16 +275,26 @@ public function test_a_gated_message_is_dead_lettered_with_its_original_bytes(): $published = [json_decode($payload, true), $queue, $payload]; }); + // The consumer's topic names the DLQ (`meta.queue` is unknown here, the gate never decodes the body). (new Dispatcher('dead_letter', 0, new DeadLetterPublisher($transport)))($message); + // Without a topic (and no Kafka-side way to derive one) a publisher cannot name `.dlq`: nothing is written to + // `default.dlq`, and the record is dropped and logged (ADR-0036 decision 3), not rethrown. + $client = Mockery::mock(KafkaConsumerClient::class); + $client->shouldReceive('receive')->once()->andReturn($this->raw(['bq-schema-version' => 'x'], self::ENVELOPE, null)); + $topicless = (new KafkaConsumer($client))->receive(); + $dropped = 0; + (new Dispatcher('dead_letter', 0, new DeadLetterPublisher(Mockery::mock(Transport::class)), onDrop: static function () use (&$dropped): void { + $dropped++; + }))($topicless); + $this->assertSame(1, $dropped); + $this->assertSame(self::ENVELOPE, $published[0]['raw']); $this->assertSame('unknown_urn', $published[0]['dead_letter']['reason']); // The sentinel URN is an in-process marker: the dead-letter record never carries it. $this->assertStringNotContainsString('"job":"babelqueue:unsupported-schema-version', $published[2]); $this->assertArrayNotHasKey('job', $published[0]); - // The gate does not decode the body, so `meta.queue` is unknown and the record goes to the - // default dead-letter queue rather than `.dlq`. - $this->assertSame('default.dlq', $published[1]); + $this->assertSame('orders.dlq', $published[1]); } // -- poison / no_identity -> .dlq with reason poison (ADR-0034 §10) ---------- @@ -433,7 +444,7 @@ public function test_an_unencodable_poison_body_is_dead_lettered_byte_exact_and_ $this->assertSame(['produce', 'commit'], $log, 'written to the DLQ and committed: the partition must not wedge'); $payload = $recorder->records[0]['payload']; - $this->assertSame('default.dlq', $recorder->records[0]['topic']); + $this->assertSame('orders.dlq', $recorder->records[0]['topic']); $dlq = json_decode($payload, true); $this->assertIsArray($dlq, 'the DLQ record is valid JSON'); @@ -465,24 +476,102 @@ public function test_a_poison_copy_preserves_empty_objects_and_big_integers(): v $this->assertSame('poison', json_decode($payload, false)->dead_letter->reason); } - public function test_the_dlq_topic_falls_back_to_the_body_queue_then_default_without_a_client_topic(): void + public function test_the_dlq_topic_is_the_clients_topic_never_the_body_queue(): void { $log = []; [$producer, $recorder] = $this->recordingProducer($log); $client = Mockery::mock(KafkaConsumerClient::class); - $client->shouldReceive('receive')->twice()->andReturn( - $this->raw([], '{"meta":{"queue":"billing"}}'), - $this->raw([], '{}'), + $client->shouldReceive('receive')->once()->andReturn( + $this->raw([], '{"meta":{"queue":"billing"}}', 'shipping'), ); - $client->shouldReceive('commit')->twice(); + $client->shouldReceive('commit')->once(); - $consumer = new KafkaConsumer($client, $producer); - $consumer->receive(); - $consumer->receive(); + (new KafkaConsumer($client, $producer))->receive(); + + $this->assertSame(['shipping.dlq'], array_column($recorder->records, 'topic')); + } + + /** + * @param list|null $subscribed null = a client that is not a {@see KafkaSubscriptionDescriptor} + * @return array{0: KafkaConsumerClient, 1: object, 2: object} client, recorder, producer + */ + private function poisonSetup(?array $subscribed, ?string $frameTopic): array + { + $log = []; + [$producer, $recorder] = $this->recordingProducer($log); + $frame = $this->raw([], '{"meta":{"queue":"billing"}}', $frameTopic); + $client = $subscribed === null + ? Mockery::mock(KafkaConsumerClient::class) + : Mockery::mock(KafkaConsumerClient::class, KafkaSubscriptionDescriptor::class); + $client->shouldReceive('receive')->once()->andReturn($frame); + $client->shouldReceive('commit')->once(); // always committed: never a wedged partition + if ($subscribed !== null) { + $client->shouldReceive('subscribedTopics')->andReturn($subscribed); + } + + return [$client, $recorder, $producer]; + } + + public function test_a_poison_record_without_any_topic_is_committed_dropped_and_logged_never_a_default_dlq(): void + { + foreach ([[null, null], [null, ''], [['a', 'b'], null], [[], null]] as [$subscribed, $frameTopic]) { + [$client, $recorder, $producer] = $this->poisonSetup($subscribed, $frameTopic); + $logged = ''; + $previous = ini_set('error_log', $file = tempnam(sys_get_temp_dir(), 'bqlog')); + + try { + $this->assertNull((new KafkaConsumer($client, $producer))->receive(), 'no throw: the partition must not wedge'); + $logged = (string) file_get_contents($file); + } finally { + ini_set('error_log', (string) $previous); + unlink($file); + } + + $this->assertSame([], $recorder->records, 'neither billing.dlq (meta.queue) nor default.dlq'); + $this->assertStringContainsString('poison Kafka record was dropped', $logged); + } + } + + public function test_the_dlq_topic_falls_back_to_the_constructor_topic_then_a_single_subscribed_topic(): void + { + // the constructor's topic + [$client, $recorder, $producer] = $this->poisonSetup(null, null); + (new KafkaConsumer($client, $producer, '.dlq', 'shipping'))->receive(); + $this->assertSame(['shipping.dlq'], array_column($recorder->records, 'topic')); + + // a single subscribed topic of the client + [$client, $recorder, $producer] = $this->poisonSetup(['invoices'], null); + (new KafkaConsumer($client, $producer))->receive(); + $this->assertSame(['invoices.dlq'], array_column($recorder->records, 'topic')); + + // the frame wins over both + [$client, $recorder, $producer] = $this->poisonSetup(['invoices'], 'orders'); + (new KafkaConsumer($client, $producer, '.dlq', 'shipping'))->receive(); + $this->assertSame(['orders.dlq'], array_column($recorder->records, 'topic')); + } + + public function test_the_message_queue_follows_the_same_derivation(): void + { + $client = Mockery::mock(KafkaConsumerClient::class, KafkaSubscriptionDescriptor::class); + $client->shouldReceive('receive')->twice()->andReturn($this->raw([], self::ENVELOPE, null), $this->raw([], self::ENVELOPE, null)); + $client->shouldReceive('subscribedTopics')->andReturn(['orders', 'billing'], ['orders']); + $consumer = new KafkaConsumer($client); + + $this->assertNull($consumer->receive()?->queue(), 'two subscribed topics: ambiguous'); + $this->assertSame('orders', $consumer->receive()?->queue()); + } + + public function test_receive_carries_the_source_topic_as_the_queue(): void + { + $client = Mockery::mock(KafkaConsumerClient::class); + $client->shouldReceive('receive')->twice()->andReturn( + $this->raw([], self::ENVELOPE, 'shipping'), + $this->raw([], self::ENVELOPE, null), + ); + $consumer = new KafkaConsumer($client); - $this->assertSame('billing.dlq', $recorder->records[0]['topic']); - $this->assertSame('default.dlq', $recorder->records[1]['topic']); - $this->assertNull($recorder->records[1]['key']); + $this->assertSame('shipping', $consumer->receive()?->queue()); + $this->assertNull($consumer->receive()?->queue()); } public function test_an_unusable_body_without_a_dlq_producer_is_committed_and_dropped(): void diff --git a/tests/KafkaKeyPreservationTest.php b/tests/KafkaKeyPreservationTest.php index 4b5a761..6d36a15 100644 --- a/tests/KafkaKeyPreservationTest.php +++ b/tests/KafkaKeyPreservationTest.php @@ -183,9 +183,10 @@ public function commit(): void $message = (new KafkaConsumer($client))->receive(); $this->assertNotNull($message); - (new Dispatcher('dead_letter', 0, new DeadLetterPublisher(new KafkaTransport($producer))))($message); + // The gate-rejected envelope has no meta.queue: the queue is the caller's (the consumer's topic). + (new DeadLetterPublisher(new KafkaTransport($producer)))->publish($message, 'unknown_urn', null, null, 'orders'); - $this->assertCarriesKey($producer->records, 'default.dlq', $key); + $this->assertCarriesKey($producer->records, 'orders.dlq', $key); } #[DataProvider('keys')] @@ -195,9 +196,9 @@ public function test_a_poison_record_keeps_its_key_to_the_dlq(?string $key): voi $raw = '{not json'; $message = new KafkaMessage(EnvelopeCodec::decode($raw, static function (): void {}), [], $raw, null, $key); - (new DeadLetterPublisher(new KafkaTransport($producer)))->publish($message, 'poison', null); + (new DeadLetterPublisher(new KafkaTransport($producer)))->publish($message, 'poison', null, null, 'orders'); - $this->assertCarriesKey($producer->records, 'default.dlq', $key); + $this->assertCarriesKey($producer->records, 'orders.dlq', $key); } public function test_a_producer_without_key_support_keeps_todays_behaviour(): void diff --git a/tests/PulsarConsumerConformanceTest.php b/tests/PulsarConsumerConformanceTest.php index 87ebc37..dbd4711 100644 --- a/tests/PulsarConsumerConformanceTest.php +++ b/tests/PulsarConsumerConformanceTest.php @@ -26,7 +26,7 @@ public function test_attempts_reconciliation_matches_the_golden_cases(): void $payload = (string) json_encode([ 'job' => 'urn:babel:orders:created', 'trace_id' => 'trace-1', - 'data' => [], + 'data' => new \stdClass(), // canonical `{}`: a list `[]` is a wrong-typed member, i.e. poison 'meta' => ['id' => 'm', 'queue' => 'orders', 'lang' => 'php', 'schema_version' => 1, 'created_at' => 1], 'attempts' => $case['body_attempts'], ]); @@ -39,7 +39,7 @@ public function test_attempts_reconciliation_matches_the_golden_cases(): void 'redeliveryCount' => $case['redelivery_count'], ]); - $message = (new PulsarConsumer($client))->receive(); + $message = (new PulsarConsumer($client, subscriptionType: 'Shared'))->receive(); $this->assertNotNull($message, $case['name']); $this->assertSame($case['expected_attempts'], $message->attempts(), $case['name']); @@ -70,7 +70,7 @@ public function test_schema_version_gate_matches_the_golden_cases(): void 'redeliveryCount' => 0, ]); - $message = (new PulsarConsumer($client))->receive(); + $message = (new PulsarConsumer($client, subscriptionType: 'Shared'))->receive(); $this->assertNotNull($message, $label); if ($case['expect'] === 'decode') { diff --git a/tests/PulsarConsumerTest.php b/tests/PulsarConsumerTest.php index 5dc6618..5cf47d9 100644 --- a/tests/PulsarConsumerTest.php +++ b/tests/PulsarConsumerTest.php @@ -41,19 +41,19 @@ private function envelopeWith(string $job, int $attempts = 0): string return (string) json_encode([ 'job' => $job, 'trace_id' => 'trace-1', - 'data' => [], + 'data' => new \stdClass(), // canonical `{}`: a list `[]` is a wrong-typed member, i.e. poison 'meta' => ['id' => 'm', 'queue' => 'orders', 'lang' => 'php', 'schema_version' => 1, 'created_at' => 1], 'attempts' => $attempts, ]); } /** @param array $properties */ - private function receiveWith(string $payload, array $properties, int $redelivery = 0, ?callable $hook = null): ?PulsarMessage + private function receiveWith(string $payload, array $properties, int $redelivery = 0, ?callable $hook = null, ?string $queue = null): ?PulsarMessage { $client = Mockery::mock(PulsarWebSocketConsumerClient::class); $client->shouldReceive('receive')->once()->andReturn($this->raw($payload, $redelivery, $properties)); - return (new PulsarConsumer($client, $hook))->receive(); + return (new PulsarConsumer($client, $hook, $queue, subscriptionType: 'Shared'))->receive(); } public function test_receive_decodes_the_envelope_and_exposes_the_inbound_view(): void @@ -61,7 +61,7 @@ public function test_receive_decodes_the_envelope_and_exposes_the_inbound_view() $client = Mockery::mock(PulsarWebSocketConsumerClient::class); $client->shouldReceive('receive')->once()->andReturn($this->raw()); - $message = (new PulsarConsumer($client))->receive(); + $message = (new PulsarConsumer($client, subscriptionType: 'Shared'))->receive(); $this->assertInstanceOf(PulsarMessage::class, $message); $this->assertSame('urn:babel:orders:created', $message->getUrn()); @@ -78,7 +78,7 @@ public function test_receive_reconciles_attempts_to_the_redelivery_count(): void $client = Mockery::mock(PulsarWebSocketConsumerClient::class); $client->shouldReceive('receive')->once()->andReturn($this->raw(redelivery: 3)); - $message = (new PulsarConsumer($client))->receive(); + $message = (new PulsarConsumer($client, subscriptionType: 'Shared'))->receive(); $this->assertSame(3, $message->attempts()); // max(body 0, redelivery 3) } @@ -88,7 +88,7 @@ public function test_receive_returns_null_when_no_message(): void $client = Mockery::mock(PulsarWebSocketConsumerClient::class); $client->shouldReceive('receive')->once()->andReturnNull(); - $this->assertNull((new PulsarConsumer($client))->receive()); + $this->assertNull((new PulsarConsumer($client, subscriptionType: 'Shared'))->receive()); } public function test_consume_acknowledges_after_a_successful_handler(): void @@ -99,7 +99,7 @@ public function test_consume_acknowledges_after_a_successful_handler(): void $handled = null; $calls = 0; - (new PulsarConsumer($client))->consume( + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume( function (PulsarMessage $m) use (&$handled): void { $handled = $m->getUrn(); }, @@ -119,7 +119,7 @@ public function test_consume_releases_when_the_handler_throws(): void $client->shouldNotReceive('acknowledge'); $calls = 0; - (new PulsarConsumer($client))->consume( + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume( function (PulsarMessage $m): void { throw new RuntimeException('boom'); }, @@ -139,7 +139,7 @@ public function test_consume_skips_idle_receives(): void $handlerCalls = 0; $calls = 0; - (new PulsarConsumer($client))->consume( + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume( function (PulsarMessage $m) use (&$handlerCalls): void { $handlerCalls++; }, @@ -156,7 +156,7 @@ public function test_receive_carries_the_raw_body_for_the_dead_letter_path(): vo $client = Mockery::mock(PulsarWebSocketConsumerClient::class); $client->shouldReceive('receive')->once()->andReturn($this->raw()); - $this->assertSame(self::ENVELOPE, (new PulsarConsumer($client))->receive()->rawBody()); + $this->assertSame(self::ENVELOPE, (new PulsarConsumer($client, subscriptionType: 'Shared'))->receive()->rawBody()); } public function test_bq_job_property_routes_even_when_the_body_names_a_different_urn(): void @@ -189,7 +189,7 @@ public function test_bq_job_property_is_trimmed_and_drives_the_dispatcher(): voi }); $calls = 0; - (new PulsarConsumer($client))->consume($dispatcher, function () use (&$calls): bool { + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume($dispatcher, function () use (&$calls): bool { return $calls++ >= 1; }); @@ -233,7 +233,7 @@ public function test_a_body_without_a_job_is_not_routed_even_when_bq_job_is_set( $handled++; }); $calls = 0; - (new PulsarConsumer($client))->consume($dispatcher, function () use (&$calls): bool { + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume($dispatcher, function () use (&$calls): bool { return $calls++ >= 1; }); @@ -298,7 +298,7 @@ public function test_a_throwing_routing_mismatch_hook_still_routes_and_acknowled $handled = null; $consumer = new PulsarConsumer($client, function (): void { throw new RuntimeException('hook failure'); - }); + }, subscriptionType: 'Shared'); $calls = 0; $consumer->consume( function (PulsarMessage $m) use (&$handled): void { @@ -467,7 +467,7 @@ public function test_a_gated_message_never_reaches_a_handler_and_releases_via_th }); $calls = 0; - (new PulsarConsumer($client))->consume(function (PulsarMessage $m) use ($dispatcher): void { + (new PulsarConsumer($client, subscriptionType: 'Shared'))->consume(function (PulsarMessage $m) use ($dispatcher): void { try { $dispatcher($m); } catch (UnknownUrnException $e) { @@ -485,7 +485,7 @@ public function test_a_gated_message_never_reaches_a_handler_and_releases_via_th public function test_a_gated_message_is_dead_lettered_with_its_original_bytes(): void { $body = $this->envelopeWith('urn:babel:orders:created'); - $message = $this->receiveWith($body, ['bq-schema-version' => 'x']); + $message = $this->receiveWith($body, ['bq-schema-version' => 'x'], 0, null, 'orders'); $published = []; $transport = Mockery::mock(Transport::class); @@ -500,8 +500,7 @@ public function test_a_gated_message_is_dead_lettered_with_its_original_bytes(): // The sentinel URN is an in-process marker: the dead-letter record never carries it. $this->assertStringNotContainsString('"job":"babelqueue:unsupported-schema-version', $published[2]); $this->assertArrayNotHasKey('job', $published[0]); - // The gate does not decode the body, so `meta.queue` is unknown and the record goes to the - // default dead-letter queue rather than `.dlq` (documented in the CHANGELOG). - $this->assertSame('default.dlq', $published[1]); + // The gate does not decode the body, so `meta.queue` is unknown: the queue is the consumer's. + $this->assertSame('orders.dlq', $published[1]); } } diff --git a/tests/PulsarDerivedRetryProducerTest.php b/tests/PulsarDerivedRetryProducerTest.php new file mode 100644 index 0000000..3dc6784 --- /dev/null +++ b/tests/PulsarDerivedRetryProducerTest.php @@ -0,0 +1,445 @@ +//`, tenant and namespace from the consumer's real topic). + * The WebSocket exchange is proven against a loopback stand-in server (tests/Support/pulsar_ws_server.php). + */ +final class PulsarDerivedRetryProducerTest extends TestCase +{ + private const BODY = '{"job":"urn:babel:orders:created","trace_id":"t-1","data":{"order_id":7},' + . '"meta":{"id":"m-1","queue":"orders","lang":"php","schema_version":1,"created_at":1},"attempts":0}'; + + /** @var resource|null */ + private $process = null; + + private string $record = ''; + + private int $port = 0; + + protected function setUp(): void + { + EnvelopeCodec::setWarningHandler(static function (): void { + }); + } + + protected function tearDown(): void + { + EnvelopeCodec::setWarningHandler(null); + if ($this->process !== null) { + proc_terminate($this->process); + proc_close($this->process); + $this->process = null; + } + if ($this->record !== '' && is_file($this->record)) { + unlink($this->record); + } + } + + private function startServer(string $mode = 'ok'): string + { + $this->record = sys_get_temp_dir() . '/bq-pulsar-ws-' . bin2hex(random_bytes(6)) . '.jsonl'; + $process = proc_open( + [PHP_BINARY, __DIR__ . '/Support/pulsar_ws_server.php', $this->record, $mode], + [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes, + ); + $this->assertIsResource($process); + $this->process = $process; + $port = fgets($pipes[1]); + if ($port === false || (int) $port === 0) { + $this->markTestSkipped('cannot start the loopback Pulsar WebSocket stand-in: ' . stream_get_contents($pipes[2])); + } + $this->port = (int) $port; + + return 'http://127.0.0.1:' . $this->port; + } + + /** @return list}> */ + private function frames(): array + { + if (! is_file($this->record)) { + return []; + } + + return array_map( + static fn (string $line): array => json_decode($line, true, 512, JSON_THROW_ON_ERROR), + array_values(array_filter(explode("\n", (string) file_get_contents($this->record)))), + ); + } + + // -- the producer --------------------------------------------------------------------------------- + + public function test_the_producer_publishes_to_the_derived_endpoint_with_base64_payload_and_properties(): void + { + $url = $this->startServer(); + + (new PulsarWebSocketProducer($url, 'secret-token'))->publish( + 'persistent://acme/ns1/orders', + self::BODY, + ['bq-job' => 'urn:babel:orders:created', 'bq-attempts' => '2'], + ); + + $frames = $this->frames(); + $this->assertCount(1, $frames); + $this->assertSame('/ws/v2/producer/persistent/acme/ns1/orders', $frames[0]['path']); + $this->assertSame('Bearer secret-token', $frames[0]['authorization']); + $this->assertSame(self::BODY, base64_decode($frames[0]['frame']['payload'], true)); + $this->assertSame(['bq-job' => 'urn:babel:orders:created', 'bq-attempts' => '2'], $frames[0]['frame']['properties']); + $this->assertArrayNotHasKey('eventTime', $frames[0]['frame']); + } + + public function test_the_producer_answers_a_ping_and_reads_a_fragmented_reply(): void + { + $url = $this->startServer('ping_fragment'); + + (new PulsarWebSocketProducer($url))->publish('persistent://public/default/orders', self::BODY, []); + + $this->assertCount(1, $this->frames()); + } + + #[DataProvider('failingServers')] + public function test_a_failed_publish_throws_and_is_never_silent(string $mode, string $message): void + { + $url = $this->startServer($mode); + + $this->expectException(BabelQueueException::class); + $this->expectExceptionMessage($message); + + (new PulsarWebSocketProducer($url, null, 3.0))->publish('persistent://public/default/orders', self::BODY, []); + } + + /** @return iterable */ + public static function failingServers(): iterable + { + yield 'a non-ok result' => ['reject', 'Topic not found']; + yield 'a refused upgrade' => ['refuse', 'upgrade was refused']; + yield 'a wrong Sec-WebSocket-Accept' => ['bad_accept', 'Sec-WebSocket-Accept']; + yield 'a close before the ack' => ['close', 'closed the connection']; + } + + public function test_an_unreachable_service_throws(): void + { + $this->expectException(BabelQueueException::class); + $this->expectExceptionMessage('Cannot connect'); + + (new PulsarWebSocketProducer('http://127.0.0.1:1', null, 1.0))->publish('persistent://public/default/orders', self::BODY, []); + } + + public function test_the_producer_needs_a_full_topic_and_a_ws_or_http_url(): void + { + try { + (new PulsarWebSocketProducer('ws://127.0.0.1:9'))->publish('orders', self::BODY, []); + $this->fail('a local name has no tenant: it must be refused, not guessed'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('not a full topic name', $e->getMessage()); + } + + $this->expectException(\InvalidArgumentException::class); + new PulsarWebSocketProducer('tcp://host:1'); + } + + public function test_the_endpoint_path_keeps_a_proxy_prefix_and_encodes_the_topic(): void + { + $producer = new PulsarWebSocketProducer('https://gw.example/pulsar/'); + + $this->assertSame( + '/pulsar/ws/v2/producer/persistent/t/ns/a%20b', + $producer->producerPath('persistent://t/ns/a b'), + ); + $this->assertSame( + '/pulsar/ws/v2/producer/non-persistent/t/ns/x', + $producer->producerPath('non-persistent://t/ns/x'), + ); + } + + // -- the consumer derives it ---------------------------------------------------------------------- + + public function test_a_client_that_only_reads_but_reports_its_service_url_gets_a_derived_retry_producer(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'persistent://acme/ns1/orders'); + $client->seed(self::BODY); + + $consumer = new PulsarConsumer($client); // no retryProducer, no subscriptionType: nothing breaks + $message = $consumer->receive(); + $this->assertNotNull($message); + + $consumer->retry($message); + + $frames = $this->frames(); + $this->assertCount(1, $frames); + $this->assertSame('/ws/v2/producer/persistent/acme/ns1/orders', $frames[0]['path'], 'tenant/namespace come from the consumer\'s real topic'); + $this->assertSame('1', $frames[0]['frame']['properties']['bq-attempts']); + $this->assertSame(['mid-1'], $client->acked); + $this->assertSame([], $client->nacked); + } + + public function test_on_exclusive_the_default_setup_reaches_the_cap_and_dead_letters(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'persistent://acme/ns1/orders', $this->record); + $client->seed(self::BODY); + + $dlq = []; + $calls = 0; + $dispatcher = (new Dispatcher(deadLetters: new DeadLetterPublisher(new class($dlq) implements Transport { + /** @param list $dlq */ + public function __construct(private array &$dlq) + { + } + + public function publish(string $payload, ?string $queue = null): ?string + { + $this->dlq[] = [$payload, (string) $queue]; + + return null; + } + })))->on('urn:babel:orders:created', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('boom'); + }); + + $polls = 0; + (new PulsarConsumer($client))->consume($dispatcher, function () use ($client, &$polls): bool { + return $client->isDrained() || ++$polls > 30; + }); + + $this->assertSame(3, $calls, 'the cap holds on an Exclusive subscription with the default setup'); + $this->assertCount(1, $dlq); + $this->assertSame('orders.dlq', $dlq[0][1]); + $this->assertSame([], $client->nacked, 'never a nack: a nack does not advance the count on Exclusive'); + $this->assertSame(['1', '2'], array_map(static fn (array $f): string => $f['frame']['properties']['bq-attempts'], $this->frames())); + } + + public function test_an_explicit_retry_producer_wins_over_the_derived_one(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'persistent://acme/ns1/orders'); + $client->seed(self::BODY); + $explicit = new class implements \BabelQueue\Transport\PulsarWebSocketClient { + /** @var list */ + public array $topics = []; + + public function publish(string $topic, string $payload, array $properties): void + { + $this->topics[] = $topic; + } + }; + + $consumer = new PulsarConsumer($client, null, null, $explicit); + $message = $consumer->receive(); + $this->assertNotNull($message); + $consumer->retry($message); + + $this->assertSame(['persistent://acme/ns1/orders'], $explicit->topics); + $this->assertSame([], $this->frames(), 'the derived producer was not used'); + } + + public function test_an_explicit_service_url_derives_the_producer_for_a_plain_consumer_client(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'persistent://acme/ns1/orders', null, false); + $client->seed(self::BODY); + + $consumer = new PulsarConsumer($client, serviceUrl: $url); + $message = $consumer->receive(); + $this->assertNotNull($message); + $consumer->retry($message); + + $this->assertCount(1, $this->frames()); + } + + public function test_a_failing_derived_publish_nacks_the_original(): void + { + $client = new EndpointConsumerClient('http://127.0.0.1:1', 'Exclusive', 'persistent://acme/ns1/orders'); + $client->seed(self::BODY); + $consumer = new PulsarConsumer($client); + $message = $consumer->receive(); + $this->assertNotNull($message); + + $log = ini_set('error_log', '/dev/null'); + try { + $consumer->retry($message); + } finally { + ini_set('error_log', (string) $log); + } + + $this->assertSame(['mid-1'], $client->nacked, 'the original is not lost: it is negatively acknowledged'); + $this->assertSame([], $client->acked); + } + + public function test_a_local_topic_name_is_not_guessed_into_a_tenant_even_with_a_service_url(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'orders'); // local name only + + try { + new PulsarConsumer($client); + $this->fail('the retry could only throw: refused at setup'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('full topic', $e->getMessage()); + } + + $this->assertSame([], $this->frames(), 'nothing was published to a guessed tenant'); + } + + public function test_the_service_auth_token_is_sent_by_the_derived_producer(): void + { + $url = $this->startServer(); + $client = new EndpointConsumerClient($url, 'Exclusive', 'persistent://acme/ns1/orders', null, false); + $client->seed(self::BODY); + + $consumer = new PulsarConsumer($client, serviceUrl: $url, serviceAuthToken: 'tok-1'); + $message = $consumer->receive(); + $this->assertNotNull($message); + $consumer->retry($message); + + $this->assertSame('Bearer tok-1', $this->frames()[0]['authorization']); + } + + public function test_fail_closed_remains_only_when_nothing_can_be_derived(): void + { + // A service URL the client does not know (empty), no producer, type unknown/Exclusive: still a setup error. + foreach (['', null] as $unknownUrl) { + try { + new PulsarConsumer(new EndpointConsumerClient($unknownUrl ?? '', 'Exclusive', 'persistent://a/b/orders')); + $this->fail('an unknown service URL cannot derive a producer'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString(PulsarServiceEndpoint::class, $e->getMessage()); + } + } + + // ... and a declared Shared subscription is still allowed to nack. + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new EndpointConsumerClient('', 'Shared', 'persistent://a/b/orders'))); + } +} + +/** A consumer client that reports its service URL (and optionally its topic/subscription) but cannot publish. */ +final class EndpointConsumerClient implements PulsarWebSocketConsumerClient, PulsarConsumerDescriptor, PulsarServiceEndpoint +{ + /** @var list, redeliveryCount: int}> */ + private array $queue = []; + + /** @var array, redeliveryCount: int}> */ + private array $inFlight = []; + + private int $next = 1; + + private int $consumedFrames = 0; + + /** @var list */ + public array $acked = []; + + /** @var list */ + public array $nacked = []; + + /** + * @param string|null $recordFile a stand-in server's record: frames published to it come back as new deliveries + * (the broker side of the loop), as on a real Pulsar topic + */ + public function __construct( + private readonly string $serviceUrl, + private readonly string $subscription, + private readonly string $topic, + private readonly ?string $recordFile = null, + private readonly bool $reportsUrl = true, + ) { + } + + public function seed(string $payload): void + { + $this->queue[] = ['messageId' => 'mid-' . $this->next++, 'payload' => $payload, 'properties' => [], 'redeliveryCount' => 0]; + } + + public function isDrained(): bool + { + $this->pullPublished(); + + return $this->queue === [] && $this->inFlight === []; + } + + private function pullPublished(): void + { + if ($this->recordFile === null || ! is_file($this->recordFile)) { + return; + } + + $lines = array_values(array_filter(explode("\n", (string) file_get_contents($this->recordFile)))); + for (; $this->consumedFrames < count($lines); $this->consumedFrames++) { + $entry = json_decode($lines[$this->consumedFrames], true); + $this->queue[] = [ + 'messageId' => 'mid-' . $this->next++, + 'payload' => (string) base64_decode($entry['frame']['payload'], true), + 'properties' => $entry['frame']['properties'], + 'redeliveryCount' => 0, // Exclusive: a redelivery would not advance it anyway + ]; + } + } + + public function receive(): ?array + { + $this->pullPublished(); + $frame = array_shift($this->queue); + if ($frame === null) { + return null; + } + $this->inFlight[$frame['messageId']] = $frame; + + return $frame; + } + + public function acknowledge(string $messageId): void + { + $this->acked[] = $messageId; + unset($this->inFlight[$messageId]); + } + + public function negativeAcknowledge(string $messageId): void + { + $this->nacked[] = $messageId; + $frame = $this->inFlight[$messageId]; + unset($this->inFlight[$messageId]); + $this->queue[] = $frame; // Exclusive / Failover: the redelivery count is NOT advanced + } + + public function topic(): string + { + return $this->topic; + } + + public function subscriptionType(): ?string + { + return $this->subscription; + } + + public function serviceUrl(): string + { + return $this->reportsUrl ? $this->serviceUrl : ''; + } + + public function authToken(): ?string + { + return null; + } +} diff --git a/tests/PulsarExhaustionConformanceTest.php b/tests/PulsarExhaustionConformanceTest.php new file mode 100644 index 0000000..c6024df --- /dev/null +++ b/tests/PulsarExhaustionConformanceTest.php @@ -0,0 +1,143 @@ + self::URN, + 'trace_id' => 'trace-1', + 'data' => new \stdClass(), + 'meta' => ['id' => 'm-1', 'queue' => 'orders', 'lang' => 'php', 'schema_version' => 1, 'created_at' => 1], + 'attempts' => $case['body_attempts'] ?? 0, + ])]; + + foreach ($bodies as $body) { + $this->assertOutcome($case, $body); + } + }); + } + + /** @param array $case */ + private function assertOutcome(array $case, string $body): void + { + $properties = isset($case['bq_attempts']) ? ['bq-attempts' => (string) $case['bq_attempts']] : []; + $client = Mockery::mock(PulsarWebSocketConsumerClient::class); + $client->shouldReceive('receive')->once()->andReturn([ + 'messageId' => 'mid-1', + 'payload' => $body, + 'properties' => $properties, + 'redeliveryCount' => $case['redelivery_count'], + ]); + $message = (new PulsarConsumer($client, null, 'orders', subscriptionType: 'Shared'))->receive(); + $this->assertNotNull($message, $case['name']); + + $writer = $case['dead_letter'] === true; + $published = []; + $transport = new class($published) implements Transport { + /** @param list $published */ + public function __construct(private array &$published) + { + } + + public function publish(string $payload, ?string $queue = null): ?string + { + $this->published[] = [$payload, (string) $queue]; + + return null; + } + }; + + $dispatcher = new Dispatcher( + (string) ($case['on_unknown_urn'] ?? 'fail'), + (int) ($case['max_attempts'] ?? 3), + $writer ? new DeadLetterPublisher($transport) : null, + onDrop: static function (): void { + }, + ); + if ($case['kind'] === 'failed') { + $dispatcher->on(self::URN, static function (): void { + throw new RuntimeException('boom'); + }); + } + + $outcome = 'drop'; + try { + $dispatcher($message); + } catch (Throwable) { + $outcome = 'retry'; + } + if ($published !== []) { + $outcome = 'dlq'; + } + + $expect = $case['expect']; + $want = ! $writer && isset($expect['outcome_no_writer']) ? $expect['outcome_no_writer'] : $expect['outcome']; + $label = $case['name'] . ' <' . substr($body, 0, 20) . '>'; + $this->assertSame($want, $outcome, $label); + + if ($outcome !== 'dlq') { + return; + } + + $this->assertCount(1, $published, $label); + [$payload, $queue] = $published[0]; + $this->assertSame('orders.dlq', $queue, $label); + $record = json_decode($payload, true, 512, JSON_THROW_ON_ERROR); + $this->assertSame($expect['reason'], $record['dead_letter']['reason'], $label); + $this->assertSame('orders', $record['dead_letter']['original_queue'], $label); + if (isset($expect['dead_letter_attempts'])) { + $this->assertSame($expect['dead_letter_attempts'], $record['dead_letter']['attempts'], $label); + } + if ($case['kind'] === 'poison' && $body === '{not json') { + $this->assertSame($body, $record['raw'], $label . ': the poison bytes are kept (ADR-0035)'); + } + } +} diff --git a/tests/PulsarRetryRepublishTest.php b/tests/PulsarRetryRepublishTest.php new file mode 100644 index 0000000..28e2fc6 --- /dev/null +++ b/tests/PulsarRetryRepublishTest.php @@ -0,0 +1,644 @@ + $dlq filled with [payload, queue] + */ + private function dlqPublisher(array &$dlq): DeadLetterPublisher + { + return new DeadLetterPublisher(new class($dlq) implements Transport { + /** @param list $dlq */ + public function __construct(private array &$dlq) + { + } + + public function publish(string $payload, ?string $queue = null): ?string + { + $this->dlq[] = [$payload, (string) $queue]; + + return null; + } + }); + } + + private function consume(FakePulsarBroker $broker, Dispatcher $dispatcher, bool $withProducer = true, int $maxPolls = 50): void + { + // The broker is both the consumer and the producer seam, so it is the derived retry producer; the + // consumer-only view hides the producer to exercise the nack-only fallback. + $consumer = $withProducer + ? new PulsarConsumer($broker, null, 'orders', topicFromFrame: true) + : new PulsarConsumer(new ConsumerOnlyView($broker), null, 'orders', null, 'public', 'default', 'Shared'); + $polls = 0; + $consumer->consume($dispatcher, function () use ($broker, &$polls, $maxPolls): bool { + return $broker->isDrained() || ++$polls > $maxPolls; + }); + } + + private function alwaysFailing(Dispatcher $dispatcher, int &$calls): Dispatcher + { + return $dispatcher->on('urn:babel:orders:created', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('boom'); + }); + } + + /** @return iterable */ + public static function subscriptionTypes(): iterable + { + yield 'Exclusive / Failover (nack does not advance the count)' => [false]; + yield 'Shared / Key_Shared (nack advances the count)' => [true]; + } + + #[\PHPUnit\Framework\Attributes\DataProvider('subscriptionTypes')] + public function test_a_failing_handler_reaches_the_cap_and_is_dead_lettered_on_every_subscription_type(bool $nackAdvancesCount): void + { + $broker = new FakePulsarBroker($nackAdvancesCount); + $broker->seed(self::BODY); + $dlq = []; + $calls = 0; + + $this->consume($broker, $this->alwaysFailing(new Dispatcher(deadLetters: $this->dlqPublisher($dlq)), $calls)); + + $this->assertSame(3, $calls, 'the handler runs exactly maxAttempts (3) times'); + $this->assertCount(1, $dlq); + $this->assertSame('orders.dlq', $dlq[0][1]); + $record = json_decode($dlq[0][0], true, 512, JSON_THROW_ON_ERROR); + $this->assertSame('failed', $record['dead_letter']['reason']); + $this->assertSame(3, $record['dead_letter']['attempts']); + $this->assertTrue($broker->isDrained()); + $this->assertSame([], $broker->nacked, 'the retry never needed a negative acknowledgement'); + + // two re-publishes to the same topic, bq-attempts 1 then 2, the body's attempts in sync + $this->assertCount(2, $broker->published); + $this->assertSame('persistent://public/default/orders', $broker->published[0]['topic']); + $this->assertSame(['1', '2'], array_map(static fn (array $p): string => $p['properties']['bq-attempts'], $broker->published)); + $this->assertSame([1, 2], array_map(static fn (array $p): int => json_decode($p['payload'], true)['attempts'], $broker->published)); + } + + public function test_a_nack_only_retry_on_a_mis_declared_subscription_loops_past_the_cap(): void + { + // Why the constructor fails closed: a nack-only retry on what is really an Exclusive/Failover subscription + // loops past the cap. Declaring Shared without a producer is the caller's explicit claim. + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY); + $dlq = []; + $calls = 0; + + $this->consume($broker, $this->alwaysFailing(new Dispatcher(deadLetters: $this->dlqPublisher($dlq)), $calls), false, 10); + + $this->assertGreaterThan(3, $calls); + $this->assertSame([], $dlq); + } + + public function test_without_a_publisher_the_message_is_acknowledged_and_dropped_at_the_cap_on_exclusive(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY); + $calls = 0; + $dropped = 0; + + $this->consume($broker, $this->alwaysFailing(new Dispatcher(onDrop: function () use (&$dropped): void { + $dropped++; + }), $calls)); + + $this->assertSame(3, $calls); + $this->assertSame(1, $dropped); + $this->assertTrue($broker->isDrained()); + } + + public function test_a_poison_body_is_republished_with_its_exact_bytes_and_an_advancing_count(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed('{not json', ['bq-job' => 'urn:babel:orders:created']); + $dropped = 0; + $dispatcher = new Dispatcher('fail', 3, onDrop: function () use (&$dropped): void { + $dropped++; + }); + + $this->consume($broker, $dispatcher); + + $this->assertCount(2, $broker->published); + foreach ($broker->published as $copy) { + $this->assertSame('{not json', $copy['payload'], 'the poison bytes are kept'); + $this->assertSame('urn:babel:orders:created', $copy['properties']['bq-job'], 'the original properties are carried'); + } + $this->assertSame(['1', '2'], array_map(static fn (array $p): string => $p['properties']['bq-attempts'], $broker->published)); + $this->assertSame(1, $dropped, 'dropped at the third delivery'); + $this->assertTrue($broker->isDrained()); + } + + public function test_a_failed_republish_negatively_acknowledges_and_does_not_acknowledge(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY); + $broker->failPublish = true; + $calls = 0; + $consumer = new PulsarConsumer($broker, null, 'orders', $broker, topicFromFrame: true); + $dispatcher = $this->alwaysFailing(new Dispatcher(), $calls); + + $file = tempnam(sys_get_temp_dir(), 'bq-log'); + $previous = ini_set('error_log', (string) $file); + try { + $message = $consumer->receive(); + $this->assertNotNull($message); + try { + $dispatcher($message); + } catch (RuntimeException) { + $consumer->retry($message); + } + } finally { + ini_set('error_log', (string) $previous); + } + + $this->assertSame(['mid-1'], $broker->nacked); + $this->assertSame([], $broker->acked); + $this->assertStringContainsString('re-publish failed', (string) file_get_contents((string) $file)); + unlink((string) $file); + } + + public function test_a_version_gated_delivery_keeps_its_gate_property_and_bytes_on_retry(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY, ['bq-schema-version' => '2']); + + $consumer = new PulsarConsumer($broker, null, 'orders', $broker, topicFromFrame: true); + $message = $consumer->receive(); + $this->assertNotNull($message); + $consumer->retry($message); + + $this->assertSame(self::BODY, $broker->published[0]['payload']); + $this->assertSame('2', $broker->published[0]['properties']['bq-schema-version']); + $this->assertSame('1', $broker->published[0]['properties']['bq-attempts']); + } + + public function test_the_consumer_derives_the_producer_and_the_topic_from_its_own_client(): void + { + $broker = new FakePulsarBroker(false, 'persistent://acme/ns1/orders', 'Exclusive'); + $broker->seed(self::BODY); + $dlq = []; + $calls = 0; + + // No queue:, no retryProducer: both come from the client (it reads the topic and can publish). + $consumer = new PulsarConsumer($broker, topicFromFrame: true); + $dispatcher = $this->alwaysFailing(new Dispatcher(deadLetters: $this->dlqPublisher($dlq)), $calls); + $polls = 0; + $consumer->consume($dispatcher, function () use ($broker, &$polls): bool { + return $broker->isDrained() || ++$polls > 50; + }); + + $this->assertSame(3, $calls); + $this->assertSame('persistent://acme/ns1/orders', $broker->published[0]['topic'], 'the full topic of the client is used as-is'); + $this->assertSame('orders.dlq', $dlq[0][1], 'the queue is the local name of the client topic'); + } + + public function test_a_local_topic_name_is_completed_with_the_tenant_and_namespace(): void + { + $broker = new FakePulsarBroker(false, 'orders', 'Failover'); + $broker->frameTopic = null; // a local name only: the tenant and namespace must then be given + $broker->seed(self::BODY); + $consumer = new PulsarConsumer($broker, null, null, null, 'acme', 'ns1', topicFromFrame: true); + + $message = $consumer->receive(); + $this->assertNotNull($message); + $this->assertSame('orders', $message->queue()); + $consumer->retry($message); + + $this->assertSame('persistent://acme/ns1/orders', $broker->published[0]['topic']); + } + + public function test_explicit_arguments_win_over_the_derived_ones(): void + { + $broker = new FakePulsarBroker(false, 'persistent://acme/ns1/orders', 'Exclusive'); + $other = new FakePulsarBroker(false); + $broker->seed(self::BODY); + + $consumer = new PulsarConsumer($broker, null, 'invoices', $other, topicFromFrame: true); + $message = $consumer->receive(); + $this->assertNotNull($message); + $this->assertSame('invoices', $message->queue()); + $consumer->retry($message); + + // `queue:` names the DLQ; the copy goes back to the topic the consumer really reads. + $this->assertSame([], $broker->published); + $this->assertSame('persistent://acme/ns1/orders', $other->published[0]['topic']); + } + + public function test_an_exclusive_or_failover_subscription_without_a_retry_producer_is_a_setup_error(): void + { + foreach (['Exclusive', 'Failover', 'failover'] as $type) { + try { + new PulsarConsumer(new ConsumerOnlyView(new FakePulsarBroker(false, 'orders', $type))); + $this->fail($type . ' without a producer must throw'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('retry producer', $e->getMessage()); + } + } + + // The type can also be passed when the client does not report it. + $silent = new ConsumerOnlyView(new FakePulsarBroker(false)); + try { + new PulsarConsumer($silent, null, 'orders', null, 'public', 'default', 'Exclusive'); + $this->fail('an explicit Exclusive without a producer must throw'); + } catch (BabelQueueException) { + $this->addToAssertionCount(1); + } + + // ... and an explicit or derived producer satisfies it. + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer($silent, null, 'orders', new FakePulsarBroker(false), 'public', 'default', 'Exclusive')); + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(false, 'orders', 'Exclusive'), topicFromFrame: true)); + } + + public function test_a_known_shared_subscription_without_a_producer_keeps_the_nack_path(): void + { + foreach (['Shared', 'Key_Shared', 'key_shared'] as $type) { + $broker = new FakePulsarBroker(true, 'orders', $type); + $broker->seed(self::BODY); + $consumer = new PulsarConsumer(new ConsumerOnlyView($broker)); + $message = $consumer->receive(); + $this->assertNotNull($message); + + $consumer->retry($message); + + $this->assertSame(['mid-1'], $broker->nacked, $type); + $this->assertSame([], $broker->published, $type); + } + } + + public function test_an_unknown_subscription_type_is_treated_as_exclusive_and_fails_closed(): void + { + // The client reports no type (null), nothing is passed, and no producer can be derived. + try { + new PulsarConsumer(new ConsumerOnlyView(new FakePulsarBroker(true, 'orders', null))); + $this->fail('an unknown type without a producer must throw'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('unknown (treated as Exclusive)', $e->getMessage()); + $this->assertStringContainsString('Shared / Key_Shared', $e->getMessage(), 'the message names the way out'); + } + + // A client that is not a descriptor at all is the same (the common legacy adapter). + $bare = new class implements PulsarWebSocketConsumerClient { + public function receive(): ?array + { + return null; + } + + public function acknowledge(string $messageId): void + { + } + + public function negativeAcknowledge(string $messageId): void + { + } + }; + try { + new PulsarConsumer($bare, null, 'orders'); + $this->fail('a legacy adapter with no producer must throw'); + } catch (BabelQueueException) { + $this->addToAssertionCount(1); + } + + // Each way out works: declare Shared, or give / derive a producer. + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer($bare, null, 'orders', null, 'public', 'default', 'Shared')); + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer($bare, null, 'orders', new FakePulsarBroker(false), topicFromFrame: true)); + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(false), topicFromFrame: true)); + } + + public function test_an_unknown_subscription_type_name_is_rejected(): void + { + $this->expectException(\InvalidArgumentException::class); + new PulsarConsumer(new FakePulsarBroker(false), null, 'orders', null, 'public', 'default', 'Durable'); + } + + public function test_a_retry_whose_topic_is_unknown_fails_loudly_instead_of_nacking_unless_shared(): void + { + foreach (['Exclusive', null] as $type) { // unknown counts as Exclusive + $broker = new FakePulsarBroker(false, null, $type); // can publish, but nobody knows the topic + $broker->frameTopic = null; + $broker->seed(self::BODY); + $consumer = new PulsarConsumer($broker, topicFromFrame: true); + $message = $consumer->receive(); + $this->assertNotNull($message); + + try { + $consumer->retry($message); + $this->fail('a retry with no topic must throw for ' . ($type ?? 'unknown')); + } catch (BabelQueueException) { + $this->assertSame([], $broker->nacked); + } + } + + // Shared: the nack path is fine without a topic. + $shared = new FakePulsarBroker(true, null, 'Shared'); + $shared->frameTopic = null; + $shared->seed(self::BODY); + $consumer = new PulsarConsumer($shared); + $message = $consumer->receive(); + $this->assertNotNull($message); + $consumer->retry($message); + $this->assertSame(['mid-1'], $shared->nacked); + } + + public function test_a_full_frame_topic_in_another_tenant_is_where_the_copy_goes(): void + { + // An adapter that is not a descriptor, whose frames carry the full topic of a non-default tenant/namespace. + $broker = new FakePulsarBroker(false); + $broker->frameTopic = 'persistent://acme/prod/orders'; + $broker->seed(self::BODY); + $consumer = new PulsarConsumer($broker, topicFromFrame: true); // no tenant:, no namespace:, no queue: + $message = $consumer->receive(); + $this->assertNotNull($message); + $this->assertSame('persistent://acme/prod/orders', $message->topic()); + + $consumer->retry($message); + + $this->assertSame('persistent://acme/prod/orders', $broker->published[0]['topic'], 'never persistent://public/default/...'); + $this->assertSame(['mid-1'], $broker->acked); + } + + public function test_a_producer_that_cannot_know_where_to_publish_is_refused_at_setup(): void + { + // A derived producer, an Exclusive-ish subscription, but a LOCAL topic name and no tenant / namespace and no + // statement that frames carry the full topic: the first retry could only throw, so the constructor does. + foreach ([ + static fn () => new PulsarConsumer(new FakePulsarBroker(false, 'orders', 'Exclusive')), + static fn () => new PulsarConsumer(new FakePulsarBroker(false), null, 'orders'), + static fn () => new PulsarConsumer(new FakePulsarBroker(false), null, 'orders', null, 'acme'), + ] as $build) { + try { + $build(); + $this->fail('must be refused at setup'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('topicFromFrame', $e->getMessage()); + } + } + + // Each way out: a full descriptor topic, queue + tenant + namespace, or the explicit frame-topic declaration. + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(false, 'persistent://acme/ns/orders', 'Exclusive'))); + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(false), null, 'orders', null, 'acme', 'ns')); + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(false), topicFromFrame: true)); + // ... and a known Shared subscription never needs the topic (a nack retries). + $this->assertInstanceOf(PulsarConsumer::class, new PulsarConsumer(new FakePulsarBroker(true, 'orders', 'Shared'))); + } + + public function test_a_local_name_without_tenant_and_namespace_is_never_guessed(): void + { + $broker = new FakePulsarBroker(false); + $broker->frameTopic = null; + $broker->seed(self::BODY); + $consumer = new PulsarConsumer($broker, null, 'orders', topicFromFrame: true); // Exclusive-ish (unknown type), producer derived + + $message = $consumer->receive(); + $this->assertNotNull($message); + + try { + $consumer->retry($message); + $this->fail('the copy must not go to a guessed public/default'); + } catch (BabelQueueException $e) { + $this->assertStringContainsString('tenant', $e->getMessage()); + } + $this->assertSame([], $broker->published); + $this->assertSame([], $broker->acked, 'the original is not acknowledged'); + } + + public function test_a_failed_dead_letter_write_nacks_and_does_not_re_publish_a_retry_copy(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY); + $calls = 0; + $failing = new DeadLetterPublisher(new class implements Transport { + public function publish(string $payload, ?string $queue = null): ?string + { + throw new RuntimeException('dlq down'); + } + }); + $dispatcher = $this->alwaysFailing(new Dispatcher('fail', 1, $failing), $calls); // cap 1: the first failure dead-letters + + $consumer = new PulsarConsumer($broker, null, 'orders', topicFromFrame: true); + $polls = 0; + $consumer->consume($dispatcher, function () use (&$polls): bool { + return ++$polls > 1; // one delivery, then stop + }); + + $this->assertSame([], $broker->published, 'no retry copy for a failed DLQ write'); + $this->assertSame([], $broker->acked); + $this->assertSame(['mid-1'], $broker->nacked, 'the original is negatively acknowledged so the write is tried again'); + } + + public function test_the_incoming_bq_job_route_is_kept_on_the_retry_copy_and_on_the_dead_letter_record(): void + { + // bq-job routes to urn:a although the body says urn:b (section 5.2 allows it). + $broker = new FakePulsarBroker(false); + $broker->seed(str_replace('urn:babel:orders:created', 'urn:babel:orders:refunded', self::BODY), ['bq-job' => 'urn:babel:orders:created']); + + $written = []; + $wsProducer = new class($written) implements PulsarWebSocketClient { + /** @param list}> $written */ + public function __construct(private array &$written) + { + } + + public function publish(string $topic, string $payload, array $properties): void + { + $this->written[] = ['topic' => $topic, 'payload' => $payload, 'properties' => $properties]; + } + }; + $dlq = new DeadLetterPublisher(new \BabelQueue\Transport\PulsarTransport($wsProducer)); + $calls = 0; + $dispatcher = $this->alwaysFailing(new Dispatcher(deadLetters: $dlq), $calls); + + $this->consume($broker, $dispatcher); + + $this->assertSame(3, $calls); + foreach ($broker->published as $copy) { + $this->assertSame('urn:babel:orders:created', $copy['properties']['bq-job'], 'the retry copy keeps the route'); + $this->assertSame('urn:babel:orders:refunded', json_decode($copy['payload'], true)['job'], 'the body is not rewritten'); + } + $this->assertCount(1, $written); + $this->assertSame('public', explode('/', $written[0]['topic'])[2]); + $this->assertStringEndsWith('/orders.dlq', $written[0]['topic']); + $this->assertSame('urn:babel:orders:created', $written[0]['properties']['bq-job'], 'the DLQ record keeps the incoming route'); + $this->assertSame((string) json_decode($written[0]['payload'], true)['attempts'], $written[0]['properties']['bq-attempts'], 'bq-attempts follows the record body, not the incoming property'); + } + + public function test_the_consume_loop_acks_a_success_without_publishing(): void + { + $broker = new FakePulsarBroker(false); + $broker->seed(self::BODY); + $seen = 0; + + $this->consume($broker, (new Dispatcher())->on('urn:babel:orders:created', function () use (&$seen): void { + $seen++; + })); + + $this->assertSame(1, $seen); + $this->assertSame([], $broker->published); + $this->assertSame(['mid-1'], $broker->acked); + } +} + +/** + * An in-memory Pulsar: a queue of frames plus the producer seam. `$nackAdvancesCount` selects the subscription + * semantics: true = Shared / Key_Shared (a nack redelivers with redeliveryCount + 1), false = Exclusive / Failover + * (a nack redelivers the same frame with the same redeliveryCount). + */ +final class FakePulsarBroker implements PulsarWebSocketConsumerClient, PulsarWebSocketClient, PulsarConsumerDescriptor +{ + /** @var list, redeliveryCount: int}> */ + private array $queue = []; + + /** @var array, redeliveryCount: int}> */ + private array $inFlight = []; + + private int $next = 1; + + /** @var list}> */ + public array $published = []; + + /** @var list */ + public array $acked = []; + + /** @var list */ + public array $nacked = []; + + public bool $failPublish = false; + + /** The `topic` key of every frame (a client adapter that adds the full topic); null = frames carry none. */ + public ?string $frameTopic = 'persistent://public/default/orders'; + + public function __construct( + private readonly bool $nackAdvancesCount, + private readonly ?string $topic = null, + private readonly ?string $subscription = null, + ) { + } + + public function topic(): string + { + return $this->topic ?? ''; + } + + public function subscriptionType(): ?string + { + return $this->subscription; + } + + /** @param array $properties */ + public function seed(string $payload, array $properties = []): void + { + $this->queue[] = ['messageId' => 'mid-' . $this->next++, 'payload' => $payload, 'properties' => $properties, 'redeliveryCount' => 0]; + } + + public function isDrained(): bool + { + return $this->queue === [] && $this->inFlight === []; + } + + public function receive(): ?array + { + $frame = array_shift($this->queue); + if ($frame === null) { + return null; + } + + $this->inFlight[$frame['messageId']] = $frame; + + return $this->frameTopic === null ? $frame : $frame + ['topic' => $this->frameTopic]; + } + + public function acknowledge(string $messageId): void + { + $this->acked[] = $messageId; + unset($this->inFlight[$messageId]); + } + + public function negativeAcknowledge(string $messageId): void + { + $this->nacked[] = $messageId; + $frame = $this->inFlight[$messageId]; + unset($this->inFlight[$messageId]); + if ($this->nackAdvancesCount) { + $frame['redeliveryCount']++; + } + $this->queue[] = $frame; + } + + public function publish(string $topic, string $payload, array $properties): void + { + if ($this->failPublish) { + throw new RuntimeException('broker unavailable'); + } + + $this->published[] = ['topic' => $topic, 'payload' => $payload, 'properties' => $properties]; + $this->queue[] = ['messageId' => 'mid-' . $this->next++, 'payload' => $payload, 'properties' => $properties, 'redeliveryCount' => 0]; + } +} + +/** Only the consumer side (and the descriptor) of a broker: no producer seam, so nothing can be derived. */ +final class ConsumerOnlyView implements PulsarWebSocketConsumerClient, PulsarConsumerDescriptor +{ + public function __construct(private readonly FakePulsarBroker $broker) + { + } + + public function receive(): ?array + { + return $this->broker->receive(); + } + + public function acknowledge(string $messageId): void + { + $this->broker->acknowledge($messageId); + } + + public function negativeAcknowledge(string $messageId): void + { + $this->broker->negativeAcknowledge($messageId); + } + + public function topic(): string + { + return $this->broker->topic(); + } + + public function subscriptionType(): ?string + { + return $this->broker->subscriptionType(); + } +} diff --git a/tests/PulsarWebSocketSubscriptionTest.php b/tests/PulsarWebSocketSubscriptionTest.php new file mode 100644 index 0000000..0e4f9e4 --- /dev/null +++ b/tests/PulsarWebSocketSubscriptionTest.php @@ -0,0 +1,238 @@ +process !== null) { + proc_terminate($this->process); + proc_close($this->process); + $this->process = null; + } + foreach ([$this->record, $this->record . '.consumer', $this->seed] as $file) { + if ($file !== '' && is_file($file)) { + unlink($file); + } + } + } + + private function startServer(string $subscriptionType = 'Exclusive', bool $seeded = true): string + { + $base = sys_get_temp_dir() . '/bq-pulsar-sub-' . bin2hex(random_bytes(6)); + $this->record = $base . '.jsonl'; + $this->seed = $base . '.seed'; + file_put_contents($this->seed, $seeded ? json_encode(['payload' => self::BODY, 'properties' => ['bq-job' => 'urn:babel:orders:created']]) . "\n" : ''); + + $process = proc_open( + [PHP_BINARY, __DIR__ . '/Support/pulsar_ws_server.php', $this->record, 'ok', $subscriptionType, $this->seed], + [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes, + ); + $this->assertIsResource($process); + $this->process = $process; + $port = fgets($pipes[1]); + if ($port === false || (int) $port === 0) { + $this->markTestSkipped('cannot start the loopback Pulsar stand-in: ' . stream_get_contents($pipes[2])); + } + + return 'http://127.0.0.1:' . (int) $port; + } + + /** @return list> */ + private function lines(string $file): array + { + return is_file($file) + ? array_map(static fn (string $l): array => json_decode($l, true, 512, JSON_THROW_ON_ERROR), array_values(array_filter(explode("\n", (string) file_get_contents($file))))) + : []; + } + + /** @return list */ + private function consumerEvents(): array + { + return array_map(static fn (array $l): string => (string) $l['event'], $this->lines($this->record . '.consumer')); + } + + private function subscription(string $url, string $type = 'Exclusive', ?string $token = null): PulsarWebSocketSubscription + { + return new PulsarWebSocketSubscription($url, self::TOPIC, 'my-sub', $type, $token, 0.3, 5.0); + } + + public function test_it_is_a_consumer_client_a_descriptor_and_a_service_endpoint(): void + { + $sub = new PulsarWebSocketSubscription('https://pulsar.example:8443/', self::TOPIC, 'my-sub', 'key_shared', 'tok'); + + $this->assertInstanceOf(PulsarWebSocketConsumerClient::class, $sub); + $this->assertInstanceOf(PulsarConsumerDescriptor::class, $sub); + $this->assertInstanceOf(PulsarServiceEndpoint::class, $sub); + $this->assertSame(self::TOPIC, $sub->topic()); + $this->assertSame('Key_Shared', $sub->subscriptionType()); + $this->assertSame('https://pulsar.example:8443/', $sub->serviceUrl()); + $this->assertSame('tok', $sub->authToken()); + } + + public function test_the_consumer_path_is_the_subscription_endpoint(): void + { + $sub = new PulsarWebSocketSubscription('wss://gw/pulsar', 'persistent://t/n/a b', 'sub 1', 'Shared', null, 1.0, 10.0, 500); + + $this->assertSame('/pulsar/ws/v2/consumer/persistent/t/n/a%20b/sub%201?subscriptionType=Shared&receiverQueueSize=500', $sub->consumerPath()); + } + + public function test_invalid_arguments_are_refused(): void + { + foreach ([ + ['tcp://h:1', self::TOPIC, 's', 'Exclusive'], + ['http://h:1', 'orders', 's', 'Exclusive'], + ['http://h:1', self::TOPIC, '', 'Exclusive'], + ['http://h:1', self::TOPIC, 's', 'Durable'], + ] as [$url, $topic, $name, $type]) { + try { + new PulsarWebSocketSubscription($url, $topic, $name, $type); + $this->fail('must be refused: ' . $url . ' ' . $topic . ' ' . $type); + } catch (\InvalidArgumentException) { + $this->addToAssertionCount(1); + } + } + } + + public function test_it_receives_decodes_and_acknowledges_a_message(): void + { + $sub = $this->subscription($this->startServer(), 'Exclusive', 'tok-9'); + + $frame = $sub->receive(); + + $this->assertNotNull($frame); + $this->assertSame(self::BODY, $frame['payload'], 'the base64 payload is decoded'); + $this->assertSame(['bq-job' => 'urn:babel:orders:created'], $frame['properties']); + $this->assertSame(0, $frame['redeliveryCount']); + $this->assertSame(self::TOPIC, $frame['topic']); + + $sub->acknowledge($frame['messageId']); + $this->assertNull($sub->receive(), 'nothing else arrives within the read window'); + + $open = $this->lines($this->record . '.consumer')[0]; + $this->assertSame('/ws/v2/consumer/persistent/acme/ns1/orders/my-sub?subscriptionType=Exclusive', $open['path']); + $this->assertSame('Bearer tok-9', $open['authorization']); + $this->assertSame(['open', 'ack'], $this->consumerEvents()); + $sub->close(); + } + + public function test_a_negative_acknowledgement_redelivers_and_only_shared_advances_the_count(): void + { + foreach (['Exclusive' => 0, 'Failover' => 0, 'Shared' => 1, 'Key_Shared' => 1] as $type => $expected) { + $this->tearDown(); + $sub = $this->subscription($this->startServer($type), $type); + + $first = $sub->receive(); + $this->assertNotNull($first); + $sub->negativeAcknowledge($first['messageId']); + $again = $sub->receive(); + + $this->assertNotNull($again, $type); + $this->assertSame($expected, $again['redeliveryCount'], $type); + $this->assertContains('nack', $this->consumerEvents(), $type); + $sub->close(); + } + } + + public function test_acknowledging_without_a_connection_throws(): void + { + $this->expectException(BabelQueueException::class); + + (new PulsarWebSocketSubscription('http://127.0.0.1:1', self::TOPIC, 's'))->acknowledge('x'); + } + + public function test_an_unreachable_service_throws_from_receive(): void + { + $this->expectException(BabelQueueException::class); + $this->expectExceptionMessage('Cannot connect'); + + (new PulsarWebSocketSubscription('http://127.0.0.1:1', self::TOPIC, 's', 'Exclusive', null, 0.1, 1.0))->receive(); + } + + public function test_plain_new_pulsar_consumer_holds_the_cap_on_an_exclusive_subscription(): void + { + $sub = $this->subscription($this->startServer('Exclusive'), 'Exclusive', 'tok-2'); + + $dlq = []; + $calls = 0; + $dispatcher = (new Dispatcher(deadLetters: new DeadLetterPublisher(new class($dlq) implements Transport { + /** @param list $dlq */ + public function __construct(private array &$dlq) + { + } + + public function publish(string $payload, ?string $queue = null): ?string + { + $this->dlq[] = [$payload, (string) $queue]; + + return null; + } + })))->on('urn:babel:orders:created', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('boom'); + }); + + $consumer = new PulsarConsumer($sub); // nothing else: no producer, no type, no URL, no tenant + $polls = 0; + $consumer->consume($dispatcher, function () use (&$dlq, &$polls): bool { + return $dlq !== [] || ++$polls > 60; + }); + + $this->assertSame(3, $calls, 'the handler ran exactly maxAttempts times'); + $this->assertCount(1, $dlq); + $this->assertSame('orders.dlq', $dlq[0][1]); + $record = json_decode($dlq[0][0], true, 512, JSON_THROW_ON_ERROR); + $this->assertSame('failed', $record['dead_letter']['reason']); + $this->assertSame(3, $record['dead_letter']['attempts']); + + $this->assertNotContains('nack', $this->consumerEvents(), 'never a nack: it would not advance the count on Exclusive'); + $producerFrames = $this->lines($this->record); + $this->assertCount(2, $producerFrames, 'two re-publishes: bq-attempts 1 and 2'); + $this->assertSame('/ws/v2/producer/persistent/acme/ns1/orders', $producerFrames[0]['path']); + $this->assertSame('Bearer tok-2', $producerFrames[0]['authorization'], 'the derived producer carries the subscription token'); + $this->assertSame(['1', '2'], array_map(static fn (array $f): string => $f['frame']['properties']['bq-attempts'], $producerFrames)); + $sub->close(); + } +} diff --git a/tests/Support/pulsar_ws_server.php b/tests/Support/pulsar_ws_server.php new file mode 100644 index 0000000..ba93cdf --- /dev/null +++ b/tests/Support/pulsar_ws_server.php @@ -0,0 +1,245 @@ + [ []] + * mode (producer connections): ok | reject | ping_fragment | refuse | bad_accept | close + * subscriptionType: how the mini broker redelivers a nack: Exclusive / Failover keep redeliveryCount, Shared / Key_Shared add 1 + * seedFile: JSON lines {"payload": "...", "properties": {...}} delivered to the first consumer connection + * + * It prints the port on its first stdout line and serves any number of connections at once until killed. + * - producer connection (/ws/v2/producer/...): each frame is appended to as {path, authorization, frame}, + * answered per mode, and (mode ok) delivered to the consumer connection like a real topic would. + * - consumer connection (/ws/v2/consumer/...): its request line and every ack / nack frame are appended to + * .consumer as {event: open|ack|nack, path|messageId, authorization}. + */ + +[$script, $recordFile, $mode, $subscriptionType, $seedFile] = $argv + [null, null, 'ok', 'Exclusive', null]; +$consumerFile = $recordFile . '.consumer'; +$server = stream_socket_server('tcp://127.0.0.1:0', $errno, $errstr); +if ($server === false) { + fwrite(STDERR, "listen failed: $errstr\n"); + exit(1); +} +fwrite(STDOUT, explode(':', stream_socket_get_name($server, false))[1] . "\n"); +fflush(STDOUT); + +/** @var array $conns */ +$conns = []; +/** @var list $pending */ +$pending = []; +/** @var array $inflight */ +$inflight = []; +$consumerId = null; +$counter = 1; + +function frameBytes(int $opcode, string $data, bool $fin = true): string +{ + $len = strlen($data); + + return chr(($fin ? 0x80 : 0) | $opcode) . ($len < 126 ? chr($len) : chr(126) . pack('n', $len)) . $data; +} + +/** @return array{0: int, 1: string, 2: int}|null opcode, payload, consumed bytes */ +function parseFrame(string $buf): ?array +{ + if (strlen($buf) < 2) { + return null; + } + $len = ord($buf[1]) & 0x7F; + $offset = 2; + if ($len === 126) { + if (strlen($buf) < 4) { + return null; + } + $len = unpack('n', substr($buf, 2, 2))[1]; + $offset = 4; + } elseif ($len === 127) { + if (strlen($buf) < 10) { + return null; + } + $len = unpack('J', substr($buf, 2, 8))[1]; + $offset = 10; + } + $masked = (ord($buf[1]) & 0x80) !== 0; + $mask = ''; + if ($masked) { + if (strlen($buf) < $offset + 4) { + return null; + } + $mask = substr($buf, $offset, 4); + $offset += 4; + } + if (strlen($buf) < $offset + $len) { + return null; + } + $data = substr($buf, $offset, $len); + if ($masked && $len > 0) { + $data ^= str_pad($mask, $len, $mask); + } + + return [ord($buf[0]) & 0x0F, $data, $offset + $len]; +} + +function deliver(array $msg, array &$conns, ?int $consumerId, array &$inflight): bool +{ + if ($consumerId === null || ! isset($conns[$consumerId])) { + return false; + } + $inflight[$msg['messageId']] = $msg; + fwrite($conns[$consumerId]['sock'], frameBytes(0x1, json_encode([ + 'messageId' => $msg['messageId'], + 'payload' => base64_encode($msg['payload']), + 'properties' => (object) $msg['properties'], + 'publishTime' => '2026-10-25T00:00:00Z', + 'redeliveryCount' => $msg['redeliveryCount'], + ]))); + + return true; +} + +while (true) { + $read = [$server]; + foreach ($conns as $c) { + $read[] = $c['sock']; + } + $w = $e = null; + if (@stream_select($read, $w, $e, 3600) === false) { + break; + } + + foreach ($read as $sock) { + if ($sock === $server) { + $new = @stream_socket_accept($server, 0); + if ($new !== false) { + $conns[(int) $new] = ['sock' => $new, 'buf' => '', 'open' => false, 'kind' => '', 'auth' => null, 'path' => '']; + } + continue; + } + + $id = (int) $sock; + $chunk = fread($sock, 65536); + if ($chunk === false || $chunk === '') { + fclose($sock); + unset($conns[$id]); + if ($consumerId === $id) { + $consumerId = null; + foreach ($inflight as $m) { // unacknowledged messages go back to the topic + array_unshift($pending, $m); + } + $inflight = []; + } + continue; + } + $conns[$id]['buf'] .= $chunk; + + if (! $conns[$id]['open']) { + $buf = $conns[$id]['buf']; + if (! str_contains($buf, "\r\n\r\n")) { + continue; + } + [$head, $rest] = explode("\r\n\r\n", $buf, 2); + preg_match('#^GET (\S+) HTTP/1.1#', $head, $p); + preg_match('/^Sec-WebSocket-Key:\s*(\S+)/mi', $head, $k); + $auth = preg_match('/^Authorization:\s*(.+?)\s*$/mi', $head, $a) ? $a[1] : null; + $conns[$id]['buf'] = $rest; + $conns[$id]['auth'] = $auth; + $conns[$id]['path'] = $p[1]; + $isConsumer = str_starts_with($p[1], '/ws/v2/consumer/') || str_contains($p[1], '/ws/v2/consumer/'); + + if (! $isConsumer && $mode === 'refuse') { + fwrite($sock, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n"); + fclose($sock); + unset($conns[$id]); + continue; + } + $accept = (! $isConsumer && $mode === 'bad_accept') ? 'AAAA' : base64_encode(sha1($k[1] . '258EAFA5-E914-47DA-95CA-C5AB0DC85B11', true)); + fwrite($sock, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: $accept\r\n\r\n"); + $conns[$id]['open'] = true; + $conns[$id]['kind'] = $isConsumer ? 'consumer' : 'producer'; + + if ($isConsumer) { + file_put_contents($consumerFile, json_encode(['event' => 'open', 'path' => $p[1], 'authorization' => $auth]) . "\n", FILE_APPEND); + $consumerId = $id; + if ($seedFile !== null && is_file($seedFile)) { + foreach (array_filter(explode("\n", (string) file_get_contents($seedFile))) as $line) { + $seed = json_decode($line, true); + $pending[] = ['messageId' => base64_encode('id-' . $counter++), 'payload' => $seed['payload'], 'properties' => $seed['properties'] ?? [], 'redeliveryCount' => 0]; + } + $seedFile = null; + } + while ($pending !== []) { + deliver(array_shift($pending), $conns, $consumerId, $inflight); + } + } + } + + while (($frame = parseFrame($conns[$id]['buf'])) !== null) { + [$opcode, $data, $used] = $frame; + $conns[$id]['buf'] = substr($conns[$id]['buf'], $used); + + if ($opcode === 0x8) { + fclose($sock); + unset($conns[$id]); + if ($consumerId === $id) { + $consumerId = null; + } + break; + } + if ($opcode !== 0x1) { + continue; + } + + if ($conns[$id]['kind'] === 'consumer') { + $msg = json_decode($data, true); + $isNack = ($msg['type'] ?? '') === 'negativeAcknowledge'; + file_put_contents($consumerFile, json_encode(['event' => $isNack ? 'nack' : 'ack', 'messageId' => $msg['messageId'] ?? null]) . "\n", FILE_APPEND); + $held = $inflight[$msg['messageId']] ?? null; + unset($inflight[$msg['messageId'] ?? '']); + if ($isNack && $held !== null) { + if (in_array($subscriptionType, ['Shared', 'Key_Shared'], true)) { + $held['redeliveryCount']++; + } + deliver($held, $conns, $consumerId, $inflight) || $pending[] = $held; + } + continue; + } + + // producer connection + $decoded = json_decode($data, true); + file_put_contents($recordFile, json_encode(['path' => $conns[$id]['path'], 'authorization' => $conns[$id]['auth'], 'frame' => $decoded]) . "\n", FILE_APPEND); + $context = $decoded['context'] ?? ''; + + if ($mode === 'close') { + fwrite($sock, frameBytes(0x8, pack('n', 1011))); + fclose($sock); + unset($conns[$id]); + break; + } + $reply = $mode === 'reject' + ? json_encode(['result' => 'send-error:3', 'errorMsg' => 'Topic not found', 'context' => $context]) + : json_encode(['result' => 'ok', 'messageId' => 'CAAQAA==', 'context' => $context]); + if ($mode === 'ping_fragment') { + fwrite($sock, frameBytes(0x9, 'hb')); + $half = intdiv(strlen($reply), 2); + fwrite($sock, frameBytes(0x1, substr($reply, 0, $half), false) . frameBytes(0x0, substr($reply, $half))); + } else { + fwrite($sock, frameBytes(0x1, $reply)); + } + + if ($mode !== 'reject') { + $new = [ + 'messageId' => base64_encode('id-' . $counter++), + 'payload' => (string) base64_decode($decoded['payload'], true), + 'properties' => $decoded['properties'] ?? [], + 'redeliveryCount' => 0, + ]; + deliver($new, $conns, $consumerId, $inflight) || $pending[] = $new; + } + } + } +} diff --git a/tests/conformance/CONFORMANCE_VERSION b/tests/conformance/CONFORMANCE_VERSION index dc1e644..bd8bf88 100644 --- a/tests/conformance/CONFORMANCE_VERSION +++ b/tests/conformance/CONFORMANCE_VERSION @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/tests/conformance/manifest.json b/tests/conformance/manifest.json index 63f7ca1..5d06e20 100644 --- a/tests/conformance/manifest.json +++ b/tests/conformance/manifest.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "conformance_version": "1.6.0", + "conformance_version": "1.7.0", "sections": { "cases": { "scope": "core" @@ -79,6 +79,12 @@ }, "poison_dlq": { "scope": "core" + }, + "identity_verdict": { + "scope": "core" + }, + "pulsar_exhaustion": { + "scope": "pulsar" } }, "exceptions": [ @@ -5798,5 +5804,816 @@ } } ] + }, + "identity_verdict": { + "description": "Body classification outside Kafka (ADR-0036 decision 4; ADR-0032 section 6; error-handling.md section 3). Scope: `core`, because the decoder verdict is the same for every binding. For each case, feed `body` (a JSON string; it is the raw message body, so a case may hold invalid JSON, a BOM or duplicate keys) to the SDK core decoder and assert the verdict `expect`: `ok` - the body decodes into an envelope with a valid identity; `poison` - it does not (a decode error, or no valid identity). On a transport other than Kafka a `poison` body is never handed to a handler: it takes the poison route (broker-bindings.md section 5.5 and error-handling.md section 7; ADR-0035 record with `reason: poison`), not the unknown-URN strategies of error-handling.md section 3, which remain for a valid envelope whose URN has no handler. The case set starts with EVERY `kafka_poison` case (same names, bodies and order, the first 92 cases), with `no_identity` mapped to `poison` and `ok` / `poison` unchanged, so the two sections stay in step; the 5 cases after them are extra rows that pin the ADR-0036 decision 4 bodies `kafka_poison` does not hold (a blank or non-string `job` next to a valid `urn`, and a tab-only `job`) (a body with a valid identity but a wrongly typed known member is `poison` in both); a CI oracle derives the first 92 cases from `kafka_poison`, fails on any difference, and checks the extra rows with its own identity rule. The duplicate-key rule (the LAST value of a repeated top-level key wins) and the identity rule are those of `kafka_poison`: a `job` that is not a string is ignored; a blank (empty or whitespace-only) string `job` falls back to `urn`; a `urn` that is not a non-blank string is ignored; when neither is valid the body has no identity. Rows that pin the ADR-0036 decision 4 table (a `job` of `\"\"`, `\" \\t\"`, `null` or `{}` next to a valid `urn` is `ok`; see also the extra rows): a valid envelope (`full valid`) and a body whose only valid identity is `urn` (`urn alias string`), a blank `job` with a valid `urn` (`job blank with valid urn alias`), a non-string `job` with a valid `urn` (`job number with valid urn alias`) are `ok`; `{}` (`empty object`), a non-string `job` without `urn` (`job number`, `job null`), a non-string `urn` without `job` (`urn alias number`), non-object top-level values (`null literal`, `empty array`, `string`, `number`) and a non-JSON or empty body (`not json`, `empty`) are `poison`. The edge that a transport routes on an out-of-band identity property (Pulsar / Kafka `bq-job`) is outside this section.", + "cases": [ + { + "name": "empty", + "body": "", + "expect": "poison" + }, + { + "name": "whitespace only", + "body": " ", + "expect": "poison" + }, + { + "name": "not json", + "body": "not-json", + "expect": "poison" + }, + { + "name": "truncated", + "body": "{\"job\":\"urn:a\"", + "expect": "poison" + }, + { + "name": "empty array", + "body": "[]", + "expect": "poison" + }, + { + "name": "array", + "body": "[1]", + "expect": "poison" + }, + { + "name": "string", + "body": "\"str\"", + "expect": "poison" + }, + { + "name": "number", + "body": "123", + "expect": "poison" + }, + { + "name": "boolean", + "body": "true", + "expect": "poison" + }, + { + "name": "null literal", + "body": "null", + "expect": "poison" + }, + { + "name": "empty object", + "body": "{}", + "expect": "poison" + }, + { + "name": "full valid", + "body": "{\"job\":\"urn:babel:orders:created\",\"trace_id\":\"t\",\"data\":{},\"meta\":{\"id\":\"i\",\"queue\":\"q\",\"lang\":\"php\",\"schema_version\":1,\"created_at\":1700000000000},\"attempts\":0}", + "expect": "ok" + }, + { + "name": "job number", + "body": "{\"job\":5}", + "expect": "poison" + }, + { + "name": "job null", + "body": "{\"job\":null}", + "expect": "poison" + }, + { + "name": "urn alias number", + "body": "{\"urn\":5}", + "expect": "poison" + }, + { + "name": "urn alias string", + "body": "{\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "urn alias padded", + "body": "{\"urn\":\" urn:x \"}", + "expect": "ok" + }, + { + "name": "trace_id number", + "body": "{\"trace_id\":7}", + "expect": "poison" + }, + { + "name": "data array", + "body": "{\"data\":[]}", + "expect": "poison" + }, + { + "name": "data string", + "body": "{\"data\":\"x\"}", + "expect": "poison" + }, + { + "name": "data null", + "body": "{\"data\":null}", + "expect": "poison" + }, + { + "name": "meta string", + "body": "{\"meta\":\"x\"}", + "expect": "poison" + }, + { + "name": "meta array", + "body": "{\"meta\":[]}", + "expect": "poison" + }, + { + "name": "meta null", + "body": "{\"meta\":null}", + "expect": "poison" + }, + { + "name": "meta.id number", + "body": "{\"meta\":{\"id\":5}}", + "expect": "poison" + }, + { + "name": "meta.queue number", + "body": "{\"meta\":{\"queue\":5}}", + "expect": "poison" + }, + { + "name": "meta.lang boolean", + "body": "{\"meta\":{\"lang\":true}}", + "expect": "poison" + }, + { + "name": "meta.created_at string", + "body": "{\"meta\":{\"created_at\":\"2026\"}}", + "expect": "poison" + }, + { + "name": "meta.created_at fractional", + "body": "{\"meta\":{\"created_at\":1.5}}", + "expect": "poison" + }, + { + "name": "meta.created_at integer", + "body": "{\"meta\":{\"created_at\":1700000000000}}", + "expect": "poison" + }, + { + "name": "meta.created_at exponent", + "body": "{\"meta\":{\"created_at\":1e3}}", + "expect": "poison" + }, + { + "name": "meta.created_at decimal zero", + "body": "{\"meta\":{\"created_at\":1.0}}", + "expect": "poison" + }, + { + "name": "meta.created_at int64 overflow", + "body": "{\"meta\":{\"created_at\":9223372036854775808}}", + "expect": "poison" + }, + { + "name": "meta.created_at int64 max", + "body": "{\"meta\":{\"created_at\":9223372036854775807}}", + "expect": "poison" + }, + { + "name": "meta.created_at negative", + "body": "{\"meta\":{\"created_at\":-1}}", + "expect": "poison" + }, + { + "name": "meta.created_at negative zero", + "body": "{\"meta\":{\"created_at\":-0}}", + "expect": "poison" + }, + { + "name": "meta.schema_version string", + "body": "{\"meta\":{\"schema_version\":\"1\"}}", + "expect": "poison" + }, + { + "name": "meta.schema_version two", + "body": "{\"meta\":{\"schema_version\":2}}", + "expect": "poison" + }, + { + "name": "meta.schema_version decimal zero", + "body": "{\"meta\":{\"schema_version\":1.0}}", + "expect": "poison" + }, + { + "name": "meta.schema_version null", + "body": "{\"meta\":{\"schema_version\":null}}", + "expect": "poison" + }, + { + "name": "meta.schema_version boolean", + "body": "{\"meta\":{\"schema_version\":true}}", + "expect": "poison" + }, + { + "name": "attempts fractional", + "body": "{\"attempts\":1.5}", + "expect": "poison" + }, + { + "name": "attempts string", + "body": "{\"attempts\":\"3\"}", + "expect": "poison" + }, + { + "name": "attempts integer", + "body": "{\"attempts\":2}", + "expect": "poison" + }, + { + "name": "attempts exponent", + "body": "{\"attempts\":1e0}", + "expect": "poison" + }, + { + "name": "attempts upper exponent", + "body": "{\"attempts\":1E2}", + "expect": "poison" + }, + { + "name": "attempts negative zero", + "body": "{\"attempts\":-0}", + "expect": "poison" + }, + { + "name": "attempts overflow", + "body": "{\"attempts\":9223372036854775808}", + "expect": "poison" + }, + { + "name": "attempts null", + "body": "{\"attempts\":null}", + "expect": "poison" + }, + { + "name": "dead_letter string", + "body": "{\"dead_letter\":\"x\"}", + "expect": "poison" + }, + { + "name": "dead_letter null", + "body": "{\"dead_letter\":null}", + "expect": "poison" + }, + { + "name": "dead_letter empty object", + "body": "{\"dead_letter\":{}}", + "expect": "poison" + }, + { + "name": "dead_letter.attempts string", + "body": "{\"dead_letter\":{\"attempts\":\"3\"}}", + "expect": "poison" + }, + { + "name": "dead_letter.reason number", + "body": "{\"dead_letter\":{\"reason\":5}}", + "expect": "poison" + }, + { + "name": "dead_letter.error number", + "body": "{\"dead_letter\":{\"error\":5}}", + "expect": "poison" + }, + { + "name": "dead_letter nullable strings", + "body": "{\"dead_letter\":{\"error\":null,\"exception\":null}}", + "expect": "poison" + }, + { + "name": "dead_letter.failed_at string", + "body": "{\"dead_letter\":{\"failed_at\":\"x\"}}", + "expect": "poison" + }, + { + "name": "dead_letter.failed_at fractional", + "body": "{\"dead_letter\":{\"failed_at\":1.5}}", + "expect": "poison" + }, + { + "name": "dead_letter.original_queue array", + "body": "{\"dead_letter\":{\"original_queue\":[]}}", + "expect": "poison" + }, + { + "name": "dead_letter.lang object", + "body": "{\"dead_letter\":{\"lang\":{}}}", + "expect": "poison" + }, + { + "name": "dead_letter full", + "body": "{\"dead_letter\":{\"error\":\"e\",\"exception\":\"x\",\"failed_at\":1,\"original_queue\":\"q\",\"attempts\":1,\"lang\":\"go\",\"reason\":\"r\"}}", + "expect": "poison" + }, + { + "name": "upper-case job key with number is an unknown member (names are case-sensitive)", + "body": "{\"JOB\":5}", + "expect": "poison" + }, + { + "name": "mixed-case meta.id key with number is an unknown member (names are case-sensitive)", + "body": "{\"Meta\":{\"ID\":5}}", + "expect": "poison" + }, + { + "name": "upper-case job key with string", + "body": "{\"JOB\":\"urn:x\"}", + "expect": "poison" + }, + { + "name": "trailing garbage", + "body": "{\"job\":\"x\"} trailing", + "expect": "poison" + }, + { + "name": "two objects", + "body": "{\"job\":\"x\"}{\"job\":\"y\"}", + "expect": "poison" + }, + { + "name": "unknown keys", + "body": "{\"x\":1}", + "expect": "poison" + }, + { + "name": "forbidden keys with odd types", + "body": "{\"timestamp\":\"x\",\"meta\":{\"max_retries\":\"x\",\"attempts\":\"y\",\"source\":5,\"ts\":\"z\"}}", + "expect": "poison" + }, + { + "name": "nested lookalikes in data", + "body": "{\"data\":{\"a\":[1,2,{\"b\":null}],\"meta\":5,\"job\":5},\"extra\":{\"meta\":5}}", + "expect": "poison" + }, + { + "name": "duplicate job string then number", + "body": "{\"job\":\"urn:x\",\"job\":5}", + "expect": "poison" + }, + { + "name": "duplicate job number then string", + "body": "{\"job\":5,\"job\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "byte order mark", + "body": "\ufeff{\"job\":\"x\"}", + "expect": "poison" + }, + { + "name": "leading zero number", + "body": "{\"attempts\":01}", + "expect": "poison" + }, + { + "name": "NaN literal", + "body": "{\"attempts\":NaN}", + "expect": "poison" + }, + { + "name": "single quotes", + "body": "{'a':1}", + "expect": "poison" + }, + { + "name": "meta duplicate-case key", + "body": "{\"meta\":{\"id\":\"a\",\"queue\":\"b\",\"lang\":\"c\",\"schema_version\":1,\"created_at\":1,\"ID\":\"dup\"}}", + "expect": "poison" + }, + { + "name": "attempts huge exponent", + "body": "{\"attempts\":1e400}", + "expect": "poison" + }, + { + "name": "data with huge number", + "body": "{\"data\":{\"n\":1e400}}", + "expect": "poison" + }, + { + "name": "job number with valid urn alias", + "body": "{\"job\":5,\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job blank with valid urn alias", + "body": "{\"job\":\" \",\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job valid with urn number", + "body": "{\"job\":\"urn:x\",\"urn\":5}", + "expect": "ok" + }, + { + "name": "job blank without urn", + "body": "{\"job\":\" \"}", + "expect": "poison" + }, + { + "name": "job and urn both empty strings", + "body": "{\"job\":\"\",\"urn\":\"\"}", + "expect": "poison" + }, + { + "name": "job and urn both arrays", + "body": "{\"job\":[],\"urn\":[]}", + "expect": "poison" + }, + { + "name": "job object without urn", + "body": "{\"job\":{}}", + "expect": "poison" + }, + { + "name": "identity with null data", + "body": "{\"job\":\"urn:x\",\"data\":null}", + "expect": "ok" + }, + { + "name": "identity with null meta", + "body": "{\"job\":\"urn:x\",\"meta\":null}", + "expect": "ok" + }, + { + "name": "identity with null attempts", + "body": "{\"job\":\"urn:x\",\"attempts\":null}", + "expect": "ok" + }, + { + "name": "identity with null dead_letter", + "body": "{\"job\":\"urn:x\",\"dead_letter\":null}", + "expect": "ok" + }, + { + "name": "identity with integer attempts", + "body": "{\"job\":\"urn:x\",\"attempts\":2}", + "expect": "ok" + }, + { + "name": "identity with wrong typed attempts", + "body": "{\"job\":\"urn:x\",\"attempts\":\"3\"}", + "expect": "poison" + }, + { + "name": "identity with wrong typed data", + "body": "{\"job\":\"urn:x\",\"data\":[]}", + "expect": "poison" + }, + { + "name": "job empty string with valid urn", + "body": "{\"job\":\"\",\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job space and tab with valid urn", + "body": "{\"job\":\" \\t\",\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job null with valid urn", + "body": "{\"job\":null,\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job object with valid urn", + "body": "{\"job\":{},\"urn\":\"urn:x\"}", + "expect": "ok" + }, + { + "name": "job space and tab without urn", + "body": "{\"job\":\" \\t\"}", + "expect": "poison" + } + ] + }, + "pulsar_exhaustion": { + "description": "Pulsar max-attempts settle decision (ADR-0036 decisions 1 and 2, amends ADR-0014; broker-bindings.md section 5.5 \"Max attempts and poison\"). Scope: `pulsar`. A pure function, no broker: for each case compute the outcome the consumer takes for a delivery and assert `expect`. Case `{name, kind, body_attempts?, bq_attempts?, redelivery_count, max_attempts?, dead_letter, on_unknown_urn?, expect}`. `kind`: `failed` (the handler throws), `unknown_urn` (an unmapped URN with no hook; strategy `fail`), `poison` (the body does not decode into an envelope, including an object with no valid identity, see `identity_verdict`). Inputs: `body_attempts` is the envelope's top-level `attempts` (absent for `poison`, which has no decodable body: its input is `bq_attempts` and `redelivery_count` only); `bq_attempts` is the `bq-attempts` message property, a string (absent = not set); `redelivery_count` is the native Pulsar RedeliveryCount (0-based); `max_attempts` absent = the default 3 (the total number of deliveries, the first included; a value below 1 is a SHOULD-level configuration error and is not a case); `dead_letter` is a boolean (dead-lettering on or off) and `off` has two meanings: in the Go and Python runtimes it is the dead-letter SETTING only, the DLQ target stays writable and the `dead_letter` strategy still writes a record; in the PHP `Consume\\Dispatcher` \"on\" means a `DeadLetterPublisher` is configured, so `off` means NO publisher and nothing can be written; `on_unknown_urn` absent = `fail`. Derivation: `reconciled` = `bq_attempts` when present and valid, else `body_attempts`, then `max(reconciled, redelivery_count)` (pulsar.attempts_reconciliation); `counted = reconciled + 1`. Decision. `failed` and `unknown_urn` (strategy `fail`): `counted < max_attempts` is `retry`; otherwise dead-lettering on is `dlq` (`reason` `failed` or `unknown_urn`, `dead_letter_attempts = counted`) and off is `drop` (acknowledge and log). `poison`: `on_unknown_urn: delete` is `drop`; `on_unknown_urn: dead_letter` is `dlq` with `reason: poison` whatever `dead_letter` says (the strategy writes the record where a DLQ writer exists; see `outcome_no_writer` below); dead-lettering on (strategy `fail` or `release`) is `dlq` with `reason: poison` and NO retry (the ADR-0035 record, `dead_letter_attempts` is not compared); dead-lettering off takes the bounded `fail` path (`release` is replaced by it) - `counted < max_attempts` is `retry`, otherwise `drop`. `unknown_urn` with `on_unknown_urn: delete` is `drop`; with `dead_letter` it is a direct `dlq` with `reason: unknown_urn` whatever `dead_letter` says (`dead_letter_attempts` is not compared; see `outcome_no_writer` below). `expect.outcome_no_writer` (optional, `drop`): present only on a row whose result differs for an SDK without a DLQ writer (the PHP `Dispatcher` without a publisher: `on_unknown_urn: dead_letter` is acknowledged, dropped and logged; owner decision 2026-10-25, ADR-0036 decision 3). A runner for such an SDK asserts `expect.outcome_no_writer` instead of `expect.outcome`; every other runner asserts `expect.outcome` and ignores the field; when the field is absent both kinds of SDK assert `expect.outcome`. On Kafka the PHP `Dispatcher` rethrows instead of dropping (outside this Pulsar-scoped section). Rows with any `on_unknown_urn` value (they are the rows with that key) apply only to an SDK that has the strategy option (the Go and Python runtimes and the PHP `Consume\\Dispatcher`, ADR-0036 decision 3); a runner for the Java, .NET or Node consumer, which have no such option, skips them and reports them as `skip`, not `pass`. `unknown_urn` with `release` is not a case: on Pulsar the Go and Python runtimes release it immediately without counting (the `unknown_urn_release_delay` known gap of broker-bindings.md section 5.5). Not covered here (no broker-less test): the DLQ write itself, the acknowledgement after a successful write, the negative acknowledgement after a failed one, and the PHP queue name of ADR-0036 decision 3; each SDK covers them in its own tests.", + "cases": [ + { + "name": "failed-reconciled-0-dlq-on-retries", + "kind": "failed", + "body_attempts": 0, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "retry" + } + }, + { + "name": "failed-reconciled-1-dlq-on-retries", + "kind": "failed", + "body_attempts": 1, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "retry" + } + }, + { + "name": "failed-reconciled-2-dlq-on-dead-letters", + "kind": "failed", + "body_attempts": 2, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "failed", + "dead_letter_attempts": 3 + } + }, + { + "name": "failed-reconciled-2-dlq-off-drops", + "kind": "failed", + "body_attempts": 2, + "redelivery_count": 0, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + }, + { + "name": "failed-reconciled-7-dlq-on-counts-past-the-cap", + "kind": "failed", + "body_attempts": 7, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "failed", + "dead_letter_attempts": 8 + } + }, + { + "name": "failed-max-1-dlq-on-first-delivery-dead-letters", + "kind": "failed", + "body_attempts": 0, + "redelivery_count": 0, + "max_attempts": 1, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "failed", + "dead_letter_attempts": 1 + } + }, + { + "name": "failed-max-1-dlq-off-first-delivery-drops", + "kind": "failed", + "body_attempts": 0, + "redelivery_count": 0, + "max_attempts": 1, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + }, + { + "name": "failed-max-5-reconciled-3-dlq-off-retries", + "kind": "failed", + "body_attempts": 3, + "redelivery_count": 0, + "max_attempts": 5, + "dead_letter": false, + "expect": { + "outcome": "retry" + } + }, + { + "name": "failed-max-5-reconciled-4-dlq-off-drops", + "kind": "failed", + "body_attempts": 4, + "redelivery_count": 0, + "max_attempts": 5, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + }, + { + "name": "unknown-urn-reconciled-1-dlq-on-retries", + "kind": "unknown_urn", + "body_attempts": 1, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "retry" + } + }, + { + "name": "unknown-urn-reconciled-2-dlq-on-dead-letters", + "kind": "unknown_urn", + "body_attempts": 2, + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "unknown_urn", + "dead_letter_attempts": 3 + } + }, + { + "name": "unknown-urn-reconciled-2-dlq-off-drops", + "kind": "unknown_urn", + "body_attempts": 2, + "redelivery_count": 0, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + }, + { + "name": "poison-reconciled-0-dlq-on-dead-letters-without-retry", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "poison" + } + }, + { + "name": "poison-reconciled-0-dlq-off-retries", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": false, + "expect": { + "outcome": "retry" + } + }, + { + "name": "poison-reconciled-2-dlq-off-drops", + "kind": "poison", + "redelivery_count": 2, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + }, + { + "name": "poison-delete-dlq-on-drops", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": true, + "on_unknown_urn": "delete", + "expect": { + "outcome": "drop" + } + }, + { + "name": "poison-delete-dlq-off-drops", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": false, + "on_unknown_urn": "delete", + "expect": { + "outcome": "drop" + } + }, + { + "name": "poison-release-dlq-on-dead-letters-without-retry", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": true, + "on_unknown_urn": "release", + "expect": { + "outcome": "dlq", + "reason": "poison" + } + }, + { + "name": "poison-release-dlq-off-reconciled-0-takes-the-bounded-fail-path", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": false, + "on_unknown_urn": "release", + "expect": { + "outcome": "retry" + } + }, + { + "name": "poison-release-dlq-off-reconciled-2-drops-not-released-forever", + "kind": "poison", + "redelivery_count": 2, + "dead_letter": false, + "on_unknown_urn": "release", + "expect": { + "outcome": "drop" + } + }, + { + "name": "poison-dead-letter-strategy-dlq-off-dead-letters", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": false, + "on_unknown_urn": "dead_letter", + "expect": { + "outcome": "dlq", + "reason": "poison", + "outcome_no_writer": "drop" + } + }, + { + "name": "unknown-urn-delete-dlq-on-drops", + "kind": "unknown_urn", + "body_attempts": 0, + "redelivery_count": 0, + "dead_letter": true, + "on_unknown_urn": "delete", + "expect": { + "outcome": "drop" + } + }, + { + "name": "unknown-urn-dead-letter-strategy-dlq-off-dead-letters-directly", + "kind": "unknown_urn", + "body_attempts": 0, + "redelivery_count": 0, + "dead_letter": false, + "on_unknown_urn": "dead_letter", + "expect": { + "outcome": "dlq", + "reason": "unknown_urn", + "outcome_no_writer": "drop" + } + }, + { + "name": "poison-dead-letter-strategy-dlq-on-dead-letters", + "kind": "poison", + "redelivery_count": 0, + "dead_letter": true, + "on_unknown_urn": "dead_letter", + "expect": { + "outcome": "dlq", + "reason": "poison" + } + }, + { + "name": "unknown-urn-dead-letter-strategy-dlq-on-dead-letters-directly", + "kind": "unknown_urn", + "body_attempts": 0, + "redelivery_count": 0, + "dead_letter": true, + "on_unknown_urn": "dead_letter", + "expect": { + "outcome": "dlq", + "reason": "unknown_urn" + } + }, + { + "name": "reconciliation-bq-attempts-string-over-body", + "kind": "failed", + "body_attempts": 0, + "bq_attempts": "2", + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "failed", + "dead_letter_attempts": 3 + } + }, + { + "name": "reconciliation-redelivery-count-floors-body", + "kind": "failed", + "body_attempts": 1, + "redelivery_count": 2, + "dead_letter": true, + "expect": { + "outcome": "dlq", + "reason": "failed", + "dead_letter_attempts": 3 + } + }, + { + "name": "reconciliation-bq-attempts-below-cap-retries", + "kind": "unknown_urn", + "body_attempts": 0, + "bq_attempts": "1", + "redelivery_count": 0, + "dead_letter": true, + "expect": { + "outcome": "retry" + } + }, + { + "name": "reconciliation-poison-bq-attempts-dlq-off-drops", + "kind": "poison", + "bq_attempts": "2", + "redelivery_count": 0, + "dead_letter": false, + "expect": { + "outcome": "drop" + } + } + ] } } diff --git a/tests/conformance/schema/results.example.json b/tests/conformance/schema/results.example.json index 9130d29..cd422f6 100644 --- a/tests/conformance/schema/results.example.json +++ b/tests/conformance/schema/results.example.json @@ -1,7 +1,7 @@ { "sdk": "example-sdk/redis", "version": "1.9.0", - "conformance_version": "1.6.0", + "conformance_version": "1.7.0", "generated_at": "2026-10-18T09:30:00Z", "bindings": [ "redis", From 3a9bf377f8df763dae10e62ee0d52e525c772071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammet=20=C5=9Eafak?= Date: Sun, 25 Oct 2026 10:16:08 +0300 Subject: [PATCH 2/2] Release v1.26.0 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7419b71..effd982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ The envelope wire format is versioned separately by `meta.schema_version` ## [Unreleased] +## [1.26.0] - 2026-10-25 + Implements ADR-0036 decisions 3, 4 and 8 for the PHP core against conformance v1.7.0 (new sections `identity_verdict` and `pulsar_exhaustion`). The envelope stays at `schema_version: 1`. This is a **MINOR** with behaviour changes, listed first.