From b8d8238a686e6b33b55c99a899a482f669b5e1d1 Mon Sep 17 00:00:00 2001 From: Alberto Ricart Date: Tue, 24 Oct 2023 11:54:13 -0500 Subject: [PATCH] [TEST] speed up parallel tests --- jetstream/tests/consmers_fetch.ts | 214 ++ jetstream/tests/consumers_consume.ts | 206 ++ jetstream/tests/consumers_next.ts | 117 + jetstream/tests/consumers_test.ts | 449 +-- .../tests/jetstream_fetchconsumer_test.ts | 611 +++ .../tests/jetstream_pullconsumer_test.ts | 1180 ++++++ .../tests/jetstream_pushconsumer_test.ts | 1432 +++++++ jetstream/tests/jetstream_test.ts | 3294 +---------------- 8 files changed, 3872 insertions(+), 3631 deletions(-) create mode 100644 jetstream/tests/consmers_fetch.ts create mode 100644 jetstream/tests/consumers_consume.ts create mode 100644 jetstream/tests/consumers_next.ts create mode 100644 jetstream/tests/jetstream_fetchconsumer_test.ts create mode 100644 jetstream/tests/jetstream_pullconsumer_test.ts create mode 100644 jetstream/tests/jetstream_pushconsumer_test.ts diff --git a/jetstream/tests/consmers_fetch.ts b/jetstream/tests/consmers_fetch.ts new file mode 100644 index 00000000..e50272a6 --- /dev/null +++ b/jetstream/tests/consmers_fetch.ts @@ -0,0 +1,214 @@ +/* + * Copyright 2022-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + cleanup, + jetstreamServerConf, + setup, +} from "../../tests/helpers/mod.ts"; +import { initStream } from "./jstest_util.ts"; +import { AckPolicy, DeliverPolicy } from "../jsapi_types.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { Empty } from "../../nats-base-client/encoders.ts"; +import { StringCodec } from "../../nats-base-client/codec.ts"; +import { deferred } from "../../nats-base-client/util.ts"; +import { assertRejects } from "https://deno.land/std@0.200.0/assert/assert_rejects.ts"; +import { nanos } from "../jsutil.ts"; +import { NatsConnectionImpl } from "../../nats-base-client/nats.ts"; +import { PullConsumerMessagesImpl } from "../consumer.ts"; +import { syncIterator } from "../../nats-base-client/core.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; + +Deno.test("consumers - fetch no messages", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "b", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const consumer = await js.consumers.get(stream, "b"); + const iter = await consumer.fetch({ + max_messages: 100, + expires: 1000, + }); + for await (const m of iter) { + m.ack(); + } + assertEquals(iter.getReceived(), 0); + assertEquals(iter.getProcessed(), 0); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - fetch less messages", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + await js.publish(subj, Empty); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "b", + ack_policy: AckPolicy.Explicit, + }); + + const consumer = await js.consumers.get(stream, "b"); + assertEquals((await consumer.info(true)).num_pending, 1); + const iter = await consumer.fetch({ expires: 1000, max_messages: 10 }); + for await (const m of iter) { + m.ack(); + } + assertEquals(iter.getReceived(), 1); + assertEquals(iter.getProcessed(), 1); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - fetch exactly messages", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + + const { stream, subj } = await initStream(nc); + const sc = StringCodec(); + const js = nc.jetstream(); + await Promise.all( + new Array(200).fill("a").map((_, idx) => { + return js.publish(subj, sc.encode(`${idx}`)); + }), + ); + + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "b", + ack_policy: AckPolicy.Explicit, + }); + + const consumer = await js.consumers.get(stream, "b"); + assertEquals((await consumer.info(true)).num_pending, 200); + + const iter = await consumer.fetch({ expires: 5000, max_messages: 100 }); + for await (const m of iter) { + m.ack(); + } + assertEquals(iter.getReceived(), 100); + assertEquals(iter.getProcessed(), 100); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - fetch deleted consumer", async () => { + const { ns, nc } = await setup(jetstreamServerConf({})); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "a", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const c = await js.consumers.get(stream, "a"); + const iter = await c.fetch({ + expires: 30000, + }); + const dr = deferred(); + setTimeout(() => { + jsm.consumers.delete(stream, "a") + .then(() => { + dr.resolve(); + }); + }, 1000); + await assertRejects( + async () => { + for await (const _m of iter) { + // nothing + } + }, + Error, + "consumer deleted", + ); + await dr; + await cleanup(ns, nc); +}); + +Deno.test("consumers - fetch listener leaks", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "messages", subjects: ["hello"] }); + + const js = nc.jetstream(); + await js.publish("hello"); + + await jsm.consumers.add("messages", { + durable_name: "myconsumer", + deliver_policy: DeliverPolicy.All, + ack_policy: AckPolicy.Explicit, + ack_wait: nanos(3000), + max_waiting: 500, + }); + + const nci = nc as NatsConnectionImpl; + const base = nci.protocol.listeners.length; + + const consumer = await js.consumers.get("messages", "myconsumer"); + + let done = false; + while (!done) { + const iter = await consumer.fetch({ + max_messages: 1, + }) as PullConsumerMessagesImpl; + for await (const m of iter) { + assertEquals(nci.protocol.listeners.length, base); + m?.nak(); + if (m.info.redeliveryCount > 100) { + done = true; + } + } + } + + assertEquals(nci.protocol.listeners.length, base); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - fetch sync", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "messages", subjects: ["hello"] }); + + const js = nc.jetstream(); + await js.publish("hello"); + await js.publish("hello"); + + await jsm.consumers.add("messages", { + durable_name: "c", + deliver_policy: DeliverPolicy.All, + ack_policy: AckPolicy.Explicit, + ack_wait: nanos(3000), + max_waiting: 500, + }); + + const consumer = await js.consumers.get("messages", "c"); + const iter = await consumer.fetch({ max_messages: 2 }); + const sync = syncIterator(iter); + assertExists(await sync.next()); + assertExists(await sync.next()); + assertEquals(await sync.next(), null); + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/consumers_consume.ts b/jetstream/tests/consumers_consume.ts new file mode 100644 index 00000000..4ba69fca --- /dev/null +++ b/jetstream/tests/consumers_consume.ts @@ -0,0 +1,206 @@ +/* + * Copyright 2022-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + cleanup, + jetstreamServerConf, + setup, +} from "../../tests/helpers/mod.ts"; +import { setupStreamAndConsumer } from "../../examples/jetstream/util.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { assertRejects } from "https://deno.land/std@0.200.0/assert/assert_rejects.ts"; +import { consumerHbTest } from "./consumers_test.ts"; +import { initStream } from "./jstest_util.ts"; +import { AckPolicy, DeliverPolicy } from "../jsapi_types.ts"; +import { deadline, deferred } from "../../nats-base-client/util.ts"; +import { nanos } from "../jsutil.ts"; +import { PullConsumerMessagesImpl } from "../consumer.ts"; +import { syncIterator } from "../../nats-base-client/core.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; + +Deno.test("consumers - consume", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + + const count = 1000; + const { stream, consumer } = await setupStreamAndConsumer(nc, count); + + const js = nc.jetstream({ timeout: 30_000 }); + const c = await js.consumers.get(stream, consumer); + const ci = await c.info(); + assertEquals(ci.num_pending, count); + const start = Date.now(); + const iter = await c.consume({ expires: 2_000, max_messages: 10 }); + for await (const m of iter) { + m.ack(); + if (m.info.pending === 0) { + const millis = Date.now() - start; + console.log( + `consumer: ${millis}ms - ${count / (millis / 1000)} msgs/sec`, + ); + break; + } + } + assertEquals(iter.getReceived(), count); + assertEquals(iter.getProcessed(), count); + assertEquals((await c.info()).num_pending, 0); + await cleanup(ns, nc); +}); + +Deno.test("consumers - consume callback rejects iter", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream, consumer } = await setupStreamAndConsumer(nc, 0); + const js = nc.jetstream(); + const c = await js.consumers.get(stream, consumer); + const iter = await c.consume({ + expires: 5_000, + max_messages: 10_000, + callback: (m) => { + m.ack(); + }, + }); + + await assertRejects( + async () => { + for await (const _o of iter) { + // should fail + } + }, + Error, + "unsupported iterator", + ); + iter.stop(); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - consume heartbeats", async () => { + await consumerHbTest(false); +}); + +Deno.test("consumers - consume deleted consumer", async () => { + const { ns, nc } = await setup(jetstreamServerConf({})); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "a", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const c = await js.consumers.get(stream, "a"); + const iter = await c.consume({ + expires: 30000, + }); + const dr = deferred(); + setTimeout(() => { + jsm.consumers.delete(stream, "a").then(() => dr.resolve()); + }, 1000); + + await assertRejects( + async () => { + for await (const _m of iter) { + // nothing + } + }, + Error, + "consumer deleted", + ); + + await dr; + await cleanup(ns, nc); +}); + +Deno.test("consumers - sub leaks consume()", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: stream, + ack_policy: AckPolicy.Explicit, + }); + //@ts-ignore: test + assertEquals(nc.protocol.subscriptions.size(), 1); + const js = nc.jetstream(); + const c = await js.consumers.get(stream, stream); + const iter = await c.consume({ expires: 30000 }); + const done = (async () => { + for await (const _m of iter) { + // nothing + } + })().then(); + setTimeout(() => { + iter.close(); + }, 1000); + + await done; + //@ts-ignore: test + assertEquals(nc.protocol.subscriptions.size(), 1); + await cleanup(ns, nc); +}); + +Deno.test("consumers - consume drain", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: stream, + ack_policy: AckPolicy.Explicit, + }); + //@ts-ignore: test + const js = nc.jetstream(); + const c = await js.consumers.get(stream, stream); + const iter = await c.consume({ expires: 30000 }); + setTimeout(() => { + nc.drain(); + }, 100); + const done = (async () => { + for await (const _m of iter) { + // nothing + } + })().then(); + + await deadline(done, 1000); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - consume sync", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "messages", subjects: ["hello"] }); + + const js = nc.jetstream(); + await js.publish("hello"); + await js.publish("hello"); + + await jsm.consumers.add("messages", { + durable_name: "c", + deliver_policy: DeliverPolicy.All, + ack_policy: AckPolicy.Explicit, + ack_wait: nanos(3000), + max_waiting: 500, + }); + + const consumer = await js.consumers.get("messages", "c"); + const iter = await consumer.consume() as PullConsumerMessagesImpl; + const sync = syncIterator(iter); + assertExists(await sync.next()); + assertExists(await sync.next()); + iter.stop(); + assertEquals(await sync.next(), null); + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/consumers_next.ts b/jetstream/tests/consumers_next.ts new file mode 100644 index 00000000..b741e2f4 --- /dev/null +++ b/jetstream/tests/consumers_next.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2022-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + cleanup, + jetstreamServerConf, + setup, +} from "../../tests/helpers/mod.ts"; +import { initStream } from "./jstest_util.ts"; +import { AckPolicy, DeliverPolicy } from "../jsapi_types.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { nanos } from "../jsutil.ts"; +import { NatsConnectionImpl } from "../../nats-base-client/nats.ts"; +import { syncIterator } from "../../nats-base-client/core.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; + +Deno.test("consumers - next", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream, subj } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: stream, + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const c = await js.consumers.get(stream, stream); + let ci = await c.info(true); + assertEquals(ci.num_pending, 0); + + let m = await c.next({ expires: 1000 }); + assertEquals(m, null); + + await Promise.all([js.publish(subj), js.publish(subj)]); + ci = await c.info(); + assertEquals(ci.num_pending, 2); + + m = await c.next(); + assertEquals(m?.seq, 1); + m?.ack(); + await nc.flush(); + + ci = await c.info(); + assertEquals(ci?.num_pending, 1); + m = await c.next(); + assertEquals(m?.seq, 2); + m?.ack(); + + await cleanup(ns, nc); +}); + +Deno.test("consumers - sub leaks next()", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: stream, + ack_policy: AckPolicy.Explicit, + }); + //@ts-ignore: test + assertEquals(nc.protocol.subscriptions.size(), 1); + const js = nc.jetstream(); + const c = await js.consumers.get(stream, stream); + await c.next({ expires: 1000 }); + //@ts-ignore: test + assertEquals(nc.protocol.subscriptions.size(), 1); + await cleanup(ns, nc); +}); + +Deno.test("consumers - next listener leaks", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "messages", subjects: ["hello"] }); + + const js = nc.jetstream(); + await js.publish("hello"); + + await jsm.consumers.add("messages", { + durable_name: "myconsumer", + deliver_policy: DeliverPolicy.All, + ack_policy: AckPolicy.Explicit, + ack_wait: nanos(3000), + max_waiting: 500, + }); + + const nci = nc as NatsConnectionImpl; + const base = nci.protocol.listeners.length; + + const consumer = await js.consumers.get("messages", "myconsumer"); + + while (true) { + const m = await consumer.next(); + if (m) { + m.nak(); + if (m.info?.redeliveryCount > 100) { + break; + } + } + } + assertEquals(nci.protocol.listeners.length, base); + + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/consumers_test.ts b/jetstream/tests/consumers_test.ts index 877e1555..c3263913 100644 --- a/jetstream/tests/consumers_test.ts +++ b/jetstream/tests/consumers_test.ts @@ -187,147 +187,6 @@ Deno.test("consumers - push consumer not supported", async () => { await cleanup(ns, nc); }); -Deno.test("consumers - fetch no messages", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "b", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const consumer = await js.consumers.get(stream, "b"); - const iter = await consumer.fetch({ - max_messages: 100, - expires: 1000, - }); - for await (const m of iter) { - m.ack(); - } - assertEquals(iter.getReceived(), 0); - assertEquals(iter.getProcessed(), 0); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - fetch less messages", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - await js.publish(subj, Empty); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "b", - ack_policy: AckPolicy.Explicit, - }); - - const consumer = await js.consumers.get(stream, "b"); - assertEquals((await consumer.info(true)).num_pending, 1); - const iter = await consumer.fetch({ expires: 1000, max_messages: 10 }); - for await (const m of iter) { - m.ack(); - } - assertEquals(iter.getReceived(), 1); - assertEquals(iter.getProcessed(), 1); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - fetch exactly messages", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - - const { stream, subj } = await initStream(nc); - const sc = StringCodec(); - const js = nc.jetstream(); - await Promise.all( - new Array(200).fill("a").map((_, idx) => { - return js.publish(subj, sc.encode(`${idx}`)); - }), - ); - - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "b", - ack_policy: AckPolicy.Explicit, - }); - - const consumer = await js.consumers.get(stream, "b"); - assertEquals((await consumer.info(true)).num_pending, 200); - - const iter = await consumer.fetch({ expires: 5000, max_messages: 100 }); - for await (const m of iter) { - m.ack(); - } - assertEquals(iter.getReceived(), 100); - assertEquals(iter.getProcessed(), 100); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - consume", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - - const count = 1000; - const { stream, consumer } = await setupStreamAndConsumer(nc, count); - - const js = nc.jetstream({ timeout: 30_000 }); - const c = await js.consumers.get(stream, consumer); - const ci = await c.info(); - assertEquals(ci.num_pending, count); - const start = Date.now(); - const iter = await c.consume({ expires: 2_000, max_messages: 10 }); - for await (const m of iter) { - m.ack(); - if (m.info.pending === 0) { - const millis = Date.now() - start; - console.log( - `consumer: ${millis}ms - ${count / (millis / 1000)} msgs/sec`, - ); - break; - } - } - assertEquals(iter.getReceived(), count); - assertEquals(iter.getProcessed(), count); - assertEquals((await c.info()).num_pending, 0); - await cleanup(ns, nc); -}); - -Deno.test("consumers - consume callback rejects iter", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream, consumer } = await setupStreamAndConsumer(nc, 0); - const js = nc.jetstream(); - const c = await js.consumers.get(stream, consumer); - const iter = await c.consume({ - expires: 5_000, - max_messages: 10_000, - callback: (m) => { - m.ack(); - }, - }); - - await assertRejects( - async () => { - for await (const _o of iter) { - // should fail - } - }, - Error, - "unsupported iterator", - ); - iter.stop(); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - consume heartbeats", async () => { - await consumerHbTest(false); -}); - Deno.test("consumers - fetch heartbeats", async () => { await consumerHbTest(true); }); @@ -365,7 +224,7 @@ async function setupDataConnCluster( return servers; } -async function consumerHbTest(fetch: boolean) { +export async function consumerHbTest(fetch: boolean) { const servers = await setupDataConnCluster(3); const nc = await connect({ port: servers[0].port }); @@ -423,73 +282,6 @@ async function consumerHbTest(fetch: boolean) { await NatsServer.stopAll(servers); } -Deno.test("consumers - fetch deleted consumer", async () => { - const { ns, nc } = await setup(jetstreamServerConf({})); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "a", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const c = await js.consumers.get(stream, "a"); - const iter = await c.fetch({ - expires: 30000, - }); - const dr = deferred(); - setTimeout(() => { - jsm.consumers.delete(stream, "a") - .then(() => { - dr.resolve(); - }); - }, 1000); - await assertRejects( - async () => { - for await (const _m of iter) { - // nothing - } - }, - Error, - "consumer deleted", - ); - await dr; - await cleanup(ns, nc); -}); - -Deno.test("consumers - consume deleted consumer", async () => { - const { ns, nc } = await setup(jetstreamServerConf({})); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "a", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const c = await js.consumers.get(stream, "a"); - const iter = await c.consume({ - expires: 30000, - }); - const dr = deferred(); - setTimeout(() => { - jsm.consumers.delete(stream, "a").then(() => dr.resolve()); - }, 1000); - - await assertRejects( - async () => { - for await (const _m of iter) { - // nothing - } - }, - Error, - "consumer deleted", - ); - - await dr; - await cleanup(ns, nc); -}); - Deno.test("consumers - bad options", async () => { const { ns, nc } = await setup(jetstreamServerConf({})); const { stream } = await initStream(nc); @@ -752,61 +544,6 @@ Deno.test("consumers - threshold_messages bytes", async () => { await cleanup(ns, nc); }); -Deno.test("consumers - next", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream, subj } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: stream, - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const c = await js.consumers.get(stream, stream); - let ci = await c.info(true); - assertEquals(ci.num_pending, 0); - - let m = await c.next({ expires: 1000 }); - assertEquals(m, null); - - await Promise.all([js.publish(subj), js.publish(subj)]); - ci = await c.info(); - assertEquals(ci.num_pending, 2); - - m = await c.next(); - assertEquals(m?.seq, 1); - m?.ack(); - await nc.flush(); - - ci = await c.info(); - assertEquals(ci?.num_pending, 1); - m = await c.next(); - assertEquals(m?.seq, 2); - m?.ack(); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - sub leaks next()", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: stream, - ack_policy: AckPolicy.Explicit, - }); - //@ts-ignore: test - assertEquals(nc.protocol.subscriptions.size(), 1); - const js = nc.jetstream(); - const c = await js.consumers.get(stream, stream); - await c.next({ expires: 1000 }); - //@ts-ignore: test - assertEquals(nc.protocol.subscriptions.size(), 1); - await cleanup(ns, nc); -}); - Deno.test("consumers - sub leaks fetch()", async () => { const { ns, nc } = await setup(jetstreamServerConf()); const { stream } = await initStream(nc); @@ -832,137 +569,6 @@ Deno.test("consumers - sub leaks fetch()", async () => { await cleanup(ns, nc); }); -Deno.test("consumers - sub leaks consume()", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: stream, - ack_policy: AckPolicy.Explicit, - }); - //@ts-ignore: test - assertEquals(nc.protocol.subscriptions.size(), 1); - const js = nc.jetstream(); - const c = await js.consumers.get(stream, stream); - const iter = await c.consume({ expires: 30000 }); - const done = (async () => { - for await (const _m of iter) { - // nothing - } - })().then(); - setTimeout(() => { - iter.close(); - }, 1000); - - await done; - //@ts-ignore: test - assertEquals(nc.protocol.subscriptions.size(), 1); - await cleanup(ns, nc); -}); - -Deno.test("consumers - consume drain", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: stream, - ack_policy: AckPolicy.Explicit, - }); - //@ts-ignore: test - const js = nc.jetstream(); - const c = await js.consumers.get(stream, stream); - const iter = await c.consume({ expires: 30000 }); - setTimeout(() => { - nc.drain(); - }, 100); - const done = (async () => { - for await (const _m of iter) { - // nothing - } - })().then(); - - await deadline(done, 1000); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - fetch listener leaks", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "messages", subjects: ["hello"] }); - - const js = nc.jetstream(); - await js.publish("hello"); - - await jsm.consumers.add("messages", { - durable_name: "myconsumer", - deliver_policy: DeliverPolicy.All, - ack_policy: AckPolicy.Explicit, - ack_wait: nanos(3000), - max_waiting: 500, - }); - - const nci = nc as NatsConnectionImpl; - const base = nci.protocol.listeners.length; - - const consumer = await js.consumers.get("messages", "myconsumer"); - - let done = false; - while (!done) { - const iter = await consumer.fetch({ - max_messages: 1, - }) as PullConsumerMessagesImpl; - for await (const m of iter) { - assertEquals(nci.protocol.listeners.length, base); - m?.nak(); - if (m.info.redeliveryCount > 100) { - done = true; - } - } - } - - assertEquals(nci.protocol.listeners.length, base); - - await cleanup(ns, nc); -}); - -Deno.test("consumers - next listener leaks", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "messages", subjects: ["hello"] }); - - const js = nc.jetstream(); - await js.publish("hello"); - - await jsm.consumers.add("messages", { - durable_name: "myconsumer", - deliver_policy: DeliverPolicy.All, - ack_policy: AckPolicy.Explicit, - ack_wait: nanos(3000), - max_waiting: 500, - }); - - const nci = nc as NatsConnectionImpl; - const base = nci.protocol.listeners.length; - - const consumer = await js.consumers.get("messages", "myconsumer"); - - while (true) { - const m = await consumer.next(); - if (m) { - m.nak(); - if (m.info?.redeliveryCount > 100) { - break; - } - } - } - assertEquals(nci.protocol.listeners.length, base); - - await cleanup(ns, nc); -}); - Deno.test("consumers - inboxPrefix is respected", async () => { const { ns, nc } = await setup(jetstreamServerConf(), { inboxPrefix: "x" }); const jsm = await nc.jetstreamManager(); @@ -990,56 +596,3 @@ Deno.test("consumers - inboxPrefix is respected", async () => { await done; await cleanup(ns, nc); }); - -Deno.test("consumers - consume sync", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "messages", subjects: ["hello"] }); - - const js = nc.jetstream(); - await js.publish("hello"); - await js.publish("hello"); - - await jsm.consumers.add("messages", { - durable_name: "c", - deliver_policy: DeliverPolicy.All, - ack_policy: AckPolicy.Explicit, - ack_wait: nanos(3000), - max_waiting: 500, - }); - - const consumer = await js.consumers.get("messages", "c"); - const iter = await consumer.consume() as PullConsumerMessagesImpl; - const sync = syncIterator(iter); - assertExists(await sync.next()); - assertExists(await sync.next()); - iter.stop(); - assertEquals(await sync.next(), null); - await cleanup(ns, nc); -}); - -Deno.test("consumers - fetch sync", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "messages", subjects: ["hello"] }); - - const js = nc.jetstream(); - await js.publish("hello"); - await js.publish("hello"); - - await jsm.consumers.add("messages", { - durable_name: "c", - deliver_policy: DeliverPolicy.All, - ack_policy: AckPolicy.Explicit, - ack_wait: nanos(3000), - max_waiting: 500, - }); - - const consumer = await js.consumers.get("messages", "c"); - const iter = await consumer.fetch({ max_messages: 2 }); - const sync = syncIterator(iter); - assertExists(await sync.next()); - assertExists(await sync.next()); - assertEquals(await sync.next(), null); - await cleanup(ns, nc); -}); diff --git a/jetstream/tests/jetstream_fetchconsumer_test.ts b/jetstream/tests/jetstream_fetchconsumer_test.ts new file mode 100644 index 00000000..85a78e9c --- /dev/null +++ b/jetstream/tests/jetstream_fetchconsumer_test.ts @@ -0,0 +1,611 @@ +/* + * Copyright 2021-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + assertBetween, + cleanup, + jetstreamServerConf, + setup, +} from "../../tests/helpers/mod.ts"; +import { initStream, time } from "./jstest_util.ts"; +import { AckPolicy, StorageType } from "../jsapi_types.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { Empty } from "../../nats-base-client/encoders.ts"; +import { assertThrows } from "https://deno.land/std@0.200.0/assert/assert_throws.ts"; +import { fail } from "https://deno.land/std@0.200.0/assert/fail.ts"; +import { assert } from "../../nats-base-client/denobuffer.ts"; +import { NatsConnectionImpl } from "../../nats-base-client/nats.ts"; +import { assertRejects } from "https://deno.land/std@0.200.0/assert/assert_rejects.ts"; +import { + DebugEvents, + Events, + NatsError, + syncIterator, +} from "../../nats-base-client/core.ts"; +import { Js409Errors } from "../jsutil.ts"; +import { nuid } from "../../nats-base-client/nuid.ts"; +import { deferred } from "../../nats-base-client/util.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; +import { consume } from "./jetstream_test.ts"; + +Deno.test("jetstream - fetch expires waits", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + const start = Date.now(); + const iter = js.fetch(stream, "me", { expires: 1000 }); + await (async () => { + for await (const _m of iter) { + // nothing + } + })(); + const elapsed = Date.now() - start; + assertBetween(elapsed, 950, 1050); + assertEquals(iter.getReceived(), 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch expires waits after initial", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + await js.publish(subj, Empty); + const start = Date.now(); + const iter = js.fetch(stream, "me", { expires: 1000, batch: 5 }); + await (async () => { + for await (const _m of iter) { + // nothing + } + })(); + const elapsed = Date.now() - start; + assertBetween(elapsed, 950, 1050); + assertEquals(iter.getReceived(), 1); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch expires or no_wait is required", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + assertThrows( + () => { + js.fetch(stream, "me"); + }, + Error, + "expires or no_wait is required", + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch: no_wait with more left", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + await js.publish(subj); + await js.publish(subj); + + const iter = js.fetch(stream, "me", { no_wait: true }); + await consume(iter); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch some messages", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + // try to get messages = none available + let sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); + await (async () => { + for await (const m of sub) { + m.ack(); + } + })(); + assertEquals(sub.getProcessed(), 0); + + // seed some messages + await js.publish(subj, Empty, { msgID: "a" }); + await js.publish(subj, Empty, { msgID: "b" }); + await js.publish(subj, Empty, { msgID: "c" }); + + // try to get 2 messages - OK + sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); + await (async () => { + for await (const m of sub) { + m.ack(); + } + })(); + assertEquals(sub.getProcessed(), 2); + + await nc.flush(); + let ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + assertEquals(ci.delivered.stream_seq, 2); + assertEquals(ci.ack_floor.stream_seq, 2); + + // try to get 2 messages - OK, but only gets 1 + sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); + await (async () => { + for await (const m of sub) { + m.ack(); + } + })(); + assertEquals(sub.getProcessed(), 1); + + await nc.flush(); + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 3); + assertEquals(ci.ack_floor.stream_seq, 3); + + // try to get 2 messages - OK, none available + sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); + await (async () => { + for await (const m of sub) { + m.ack(); + } + })(); + assertEquals(sub.getProcessed(), 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch none - breaks after expires", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const sw = time(); + const batch = js.fetch(stream, "me", { + batch: 10, + expires: 1000, + }); + const done = (async () => { + for await (const m of batch) { + console.log(m.info); + fail("expected no messages"); + } + })(); + + await done; + sw.mark(); + sw.assertInRange(1000); + assertEquals(batch.getReceived(), 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch none - no wait breaks fast", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const sw = time(); + const batch = js.fetch(stream, "me", { + batch: 10, + no_wait: true, + }); + const done = (async () => { + for await (const m of batch) { + m.ack(); + } + })(); + + await done; + sw.mark(); + assertBetween(sw.duration(), 0, 500); + assertEquals(batch.getReceived(), 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch one - no wait breaks fast", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + await js.publish(subj); + + const sw = time(); + const batch = js.fetch(stream, "me", { + batch: 10, + no_wait: true, + }); + const done = (async () => { + for await (const m of batch) { + m.ack(); + } + })(); + + await done; + sw.mark(); + console.log({ duration: sw.duration() }); + const duration = sw.duration(); + assert(150 > duration, `${duration}`); + assertEquals(batch.getReceived(), 1); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch none - cancel timers", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const sw = time(); + const batch = js.fetch(stream, "me", { + batch: 10, + expires: 1000, + }); + const done = (async () => { + for await (const m of batch) { + m.ack(); + } + })(); + + const nci = nc as NatsConnectionImpl; + const last = nci.protocol.subscriptions.sidCounter; + const sub = nci.protocol.subscriptions.get(last); + assert(sub); + sub.unsubscribe(); + + await done; + sw.mark(); + assert(25 > sw.duration()); + assertEquals(batch.getReceived(), 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch one - breaks after expires", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + await js.publish(subj); + + const sw = time(); + const batch = js.fetch(stream, "me", { + batch: 10, + expires: 1000, + }); + const done = (async () => { + for await (const m of batch) { + m.ack(); + } + })(); + + await done; + sw.mark(); + sw.assertInRange(1000); + assertEquals(batch.getReceived(), 1); + await cleanup(ns, nc); +}); + +// Deno.test("jetstream - cross account fetch", async () => { +// const { ns, nc: admin } = await setup( +// jetstreamExportServerConf(), +// { +// user: "js", +// pass: "js", +// }, +// ); +// +// // add a stream +// const { stream, subj } = await initStream(admin); +// const admjs = admin.jetstream(); +// await admjs.publish(subj, Empty, {msgID: "1"}); +// await admjs.publish(subj, Empty, {msgID: "2"}); +// +// const admjsm = await admin.jetstreamManager(); +// +// // create a durable config +// const bo = consumerOpts() as ConsumerOptsBuilderImpl; +// bo.manualAck(); +// bo.ackExplicit(); +// bo.durable("me"); +// bo.maxAckPending(10); +// const opts = bo.getOpts(); +// await admjsm.consumers.add(stream, opts.config); +// +// const nc = await connect({ +// port: ns.port, +// user: "a", +// pass: "s3cret", +// inboxPrefix: "A", +// debug: true, +// }); +// +// // the api prefix is not used for pull/fetch() +// const js = nc.jetstream({ apiPrefix: "IPA" }); +// let iter = js.fetch(stream, "me", { batch: 20, expires: 1000 }); +// const msgs = await consume(iter); +// +// assertEquals(msgs.length, 2); +// +// // msg = await js.pull(stream, "me"); +// // assertEquals(msg.seq, 2); +// // await assertThrowsAsync(async () => { +// // await js.pull(stream, "me"); +// // }, Error, "No Messages"); +// +// await cleanup(ns, admin, nc); +// }); + +Deno.test("jetstream - idleheartbeat missed on fetch", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const iter = js.fetch(stream, "me", { + expires: 2000, + idle_heartbeat: 250, + //@ts-ignore: testing + delay_heartbeat: true, + }); + + await assertRejects( + async () => { + for await (const _m of iter) { + // no message expected + } + }, + NatsError, + Js409Errors.IdleHeartbeatMissed, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - idleheartbeat on fetch", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + const iter = js.fetch(stream, "me", { + expires: 2000, + idle_heartbeat: 250, + }); + + // we don't expect this to throw + await (async () => { + for await (const _m of iter) { + // no message expected + } + })(); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch on stopped server doesn't close client", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + (async () => { + let reconnects = 0; + for await (const s of nc.status()) { + switch (s.type) { + case DebugEvents.Reconnecting: + reconnects++; + if (reconnects === 2) { + ns.restart().then((s) => { + ns = s; + }); + } + break; + case Events.Reconnect: + setTimeout(() => { + loop = false; + }); + break; + default: + // nothing + } + } + })().then(); + const jsm = await nc.jetstreamManager(); + const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); + const { name: stream } = si.config; + await jsm.consumers.add(stream, { + durable_name: "dur", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + + setTimeout(() => { + ns.stop(); + }, 2000); + + let loop = true; + while (true) { + try { + const iter = js.fetch(stream, "dur", { batch: 1, expires: 500 }); + for await (const m of iter) { + m.ack(); + } + if (!loop) { + break; + } + } catch (err) { + fail(`shouldn't have errored: ${err.message}`); + } + } + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch heartbeat", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + + const d = deferred(); + (async () => { + for await (const s of nc.status()) { + if (s.type === Events.Reconnect) { + // if we reconnect, close the client + d.resolve(); + } + } + })().then(); + + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); + const js = nc.jetstream(); + await ns.stop(); + + const iter = js.fetch("my-stream", "dur", { + batch: 1, + expires: 5000, + idle_heartbeat: 500, + }); + + await assertRejects( + async () => { + for await (const m of iter) { + m.ack(); + } + }, + Error, + "idle heartbeats missed", + ); + ns = await ns.restart(); + // this here because otherwise get a resource leak error in the test + await d; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch consumer deleted", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const d = deferred(); + const js = nc.jetstream(); + + const iter = js.fetch(name, name, { expires: 5000 }); + (async () => { + for await (const _m of iter) { + // nothing + } + })().catch((err) => { + d.resolve(err); + }); + await nc.flush(); + await jsm.consumers.delete(name, name); + + const err = await d; + assertEquals(err?.message, "consumer deleted"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - fetch sync", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + + await js.publish(name); + await js.publish(name); + + const iter = js.fetch(name, name, { batch: 2, no_wait: true }); + const sync = syncIterator(iter); + assertExists(await sync.next()); + assertExists(await sync.next()); + assertEquals(await sync.next(), null); + + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/jetstream_pullconsumer_test.ts b/jetstream/tests/jetstream_pullconsumer_test.ts new file mode 100644 index 00000000..24a04513 --- /dev/null +++ b/jetstream/tests/jetstream_pullconsumer_test.ts @@ -0,0 +1,1180 @@ +/* + * Copyright 2021-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + cleanup, + disabled, + jetstreamExportServerConf, + jetstreamServerConf, + notCompatible, + setup, +} from "../../tests/helpers/mod.ts"; +import { initStream } from "./jstest_util.ts"; +import { + AckPolicy, + ConsumerConfig, + DeliverPolicy, + StorageType, +} from "../jsapi_types.ts"; +import { assertRejects } from "https://deno.land/std@0.200.0/assert/assert_rejects.ts"; +import { Empty } from "../../nats-base-client/encoders.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { checkJsError, nanos } from "../jsutil.ts"; +import { JSONCodec, StringCodec } from "../../nats-base-client/codec.ts"; +import { + consumerOpts, + ConsumerOptsBuilderImpl, + JetStreamSubscriptionInfoable, + PubAck, +} from "../types.ts"; +import { assert } from "../../nats-base-client/denobuffer.ts"; +import { deferred, delay } from "../../nats-base-client/util.ts"; +import { + DebugEvents, + ErrorCode, + Events, + NatsError, + syncIterator, +} from "../../nats-base-client/core.ts"; +import { fail } from "https://deno.land/std@0.200.0/assert/fail.ts"; +import { JsMsg } from "../jsmsg.ts"; +import { connect } from "../../src/connect.ts"; +import { NatsConnectionImpl } from "../../nats-base-client/nats.ts"; +import { JetStreamClientImpl } from "../jsclient.ts"; +import { assertThrows } from "https://deno.land/std@0.200.0/assert/assert_throws.ts"; +import { nuid } from "../../nats-base-client/nuid.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; +import { assertArrayIncludes } from "https://deno.land/std@0.200.0/assert/assert_array_includes.ts"; +import { callbackConsume } from "./jetstream_test.ts"; + +Deno.test("jetstream - pull no messages", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + await assertRejects( + async () => { + await js.pull(stream, "me"); + }, + Error, + "no messages", + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + await js.publish(subj, Empty, { msgID: "a" }); + let ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + + const jm = await js.pull(stream, "me"); + jm.ack(); + await nc.flush(); + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 1); + assertEquals(ci.ack_floor.stream_seq, 1, JSON.stringify(ci)); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull consumer options", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + const v = await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + max_batch: 10, + max_expires: nanos(20000), + }); + + assertEquals(v.config.max_batch, 10); + assertEquals(v.config.max_expires, nanos(20000)); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub - attached iterator", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const jc = JSONCodec(); + + let sum = 0; + const opts = consumerOpts(); + opts.durable("me"); + + const js = nc.jetstream(); + const sub = await js.pullSubscribe(subj, opts); + (async () => { + for await (const msg of sub) { + assert(msg); + //@ts-ignore: test + const ne = checkJsError(msg.msg); + if (ne) { + console.log(ne.message); + } + const n = jc.decode(msg.data); + sum += n; + msg.ack(); + } + })().then(); + sub.pull({ expires: 500, batch: 5 }); + + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, true); + await delay(250); + assertEquals(sum, 0); + + let ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 0); + assertEquals(ci.ack_floor.stream_seq, 0); + + await js.publish(subj, jc.encode(1), { msgID: "1" }); + await js.publish(subj, jc.encode(2), { msgID: "2" }); + sub.pull({ expires: 500, batch: 5 }); + await delay(500); + assertEquals(sum, 3); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 2); + assertEquals(ci.ack_floor.stream_seq, 2); + + await js.publish(subj, jc.encode(3), { msgID: "3" }); + await js.publish(subj, jc.encode(5), { msgID: "4" }); + sub.pull({ expires: 500, batch: 5 }); + await delay(1000); + assertEquals(sum, 11); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 4); + assertEquals(ci.ack_floor.stream_seq, 4); + + await js.publish(subj, jc.encode(7), { msgID: "5" }); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub - attached callback", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const jc = JSONCodec(); + + let sum = 0; + const opts = consumerOpts(); + opts.durable("me"); + + opts.callback((err, msg) => { + if (err) { + switch (err.code) { + case ErrorCode.JetStream408RequestTimeout: + case ErrorCode.JetStream409: + case ErrorCode.JetStream404NoMessages: + return; + default: + fail(err.code); + } + } + if (msg) { + const n = jc.decode(msg.data); + sum += n; + msg.ack(); + } + }); + + const js = nc.jetstream(); + const sub = await js.pullSubscribe(subj, opts); + sub.pull({ expires: 500, batch: 5 }); + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, true); + await delay(250); + assertEquals(sum, 0); + + let ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 0); + assertEquals(ci.ack_floor.stream_seq, 0); + + await js.publish(subj, jc.encode(1), { msgID: "1" }); + await js.publish(subj, jc.encode(2), { msgID: "2" }); + sub.pull({ expires: 500, batch: 5 }); + await delay(500); + assertEquals(sum, 3); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 2); + assertEquals(ci.ack_floor.stream_seq, 2); + + await js.publish(subj, jc.encode(3), { msgID: "3" }); + await js.publish(subj, jc.encode(5), { msgID: "4" }); + sub.pull({ expires: 500, batch: 5 }); + await delay(1000); + assertEquals(sum, 11); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 4); + assertEquals(ci.ack_floor.stream_seq, 4); + + await js.publish(subj, jc.encode(7), { msgID: "5" }); + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub - not attached callback", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj); + + const opts = consumerOpts(); + opts.durable("me"); + opts.ackExplicit(); + opts.maxMessages(1); + opts.callback(callbackConsume(false)); + + const sub = await js.pullSubscribe(subj, opts); + sub.pull(); + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, false); + await sub.closed; + + const jsm = await nc.jetstreamManager(); + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 1); + assertEquals(ci.ack_floor.stream_seq, 1); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub requires explicit", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + + await assertRejects( + async () => { + const opts = consumerOpts(); + opts.durable("me"); + opts.ackAll(); + await js.pullSubscribe(subj, opts); + }, + Error, + "ack policy for pull", + undefined, + ); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub ephemeral", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj); + + const d = deferred(); + const opts = consumerOpts(); + + opts.ackExplicit(); + opts.callback((err, msg) => { + if (err) { + d.reject(err); + } else { + d.resolve(msg!); + } + }); + + const ps = await js.pullSubscribe(subj, opts); + ps.pull({ no_wait: true }); + const r = await d; + assertEquals(r.subject, subj); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull consumer info without pull", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + await js.publish(subj); + + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + + const sopts = consumerOpts(); + sopts.durable("me"); + await assertRejects( + async () => { + await js.subscribe(subj, sopts); + }, + Error, + "push consumer requires deliver_subject", + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - cross account pull subscribe", () => { + disabled("cross account pull subscribe test needs updating"); + // const { ns, nc: admin } = await setup( + // jetstreamExportServerConf(), + // { + // user: "js", + // pass: "js", + // }, + // ); + // + // // add a stream + // const { stream, subj } = await initStream(admin); + // const adminjs = admin.jetstream(); + // await adminjs.publish(subj); + // await adminjs.publish(subj); + // + // // FIXME: create a durable config + // const bo = consumerOpts() as ConsumerOptsBuilderImpl; + // bo.manualAck(); + // bo.ackExplicit(); + // bo.maxMessages(2); + // bo.durable("me"); + // + // // pull subscriber stalls + // const nc = await connect({ + // port: ns.port, + // user: "a", + // pass: "s3cret", + // inboxPrefix: "A", + // }); + // const js = nc.jetstream({ apiPrefix: "IPA" }); + // + // const opts = bo.getOpts(); + // const sub = await js.pullSubscribe(subj, opts); + // const done = (async () => { + // for await (const m of sub) { + // m.ack(); + // } + // })(); + // sub.pull({ batch: 2 }); + // await done; + // assertEquals(sub.getProcessed(), 2); + // + // const ci = await sub.consumerInfo(); + // assertEquals(ci.num_pending, 0); + // assertEquals(ci.delivered.stream_seq, 2); + // + // await sub.destroy(); + // await assertThrowsAsync( + // async () => { + // await sub.consumerInfo(); + // }, + // Error, + // "consumer not found", + // ); + // + // await cleanup(ns, admin, nc); +}); + +Deno.test("jetstream - cross account pull", async () => { + const { ns, nc: admin } = await setup( + jetstreamExportServerConf(), + { + user: "js", + pass: "js", + }, + ); + + // add a stream + const { stream, subj } = await initStream(admin); + const admjs = admin.jetstream(); + await admjs.publish(subj); + await admjs.publish(subj); + + const admjsm = await admin.jetstreamManager(); + + // create a durable config + const bo = consumerOpts() as ConsumerOptsBuilderImpl; + bo.manualAck(); + bo.ackExplicit(); + bo.durable("me"); + const opts = bo.getOpts(); + await admjsm.consumers.add(stream, opts.config); + + const nc = await connect({ + port: ns.port, + user: "a", + pass: "s3cret", + inboxPrefix: "A", + }); + + // the api prefix is not used for pull/fetch() + const js = nc.jetstream({ apiPrefix: "IPA" }); + let msg = await js.pull(stream, "me"); + assertEquals(msg.seq, 1); + msg = await js.pull(stream, "me"); + assertEquals(msg.seq, 2); + await assertRejects( + async () => { + await js.pull(stream, "me"); + }, + Error, + "no messages", + undefined, + ); + + await cleanup(ns, admin, nc); +}); + +Deno.test("jetstream - pull stream doesn't exist", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const js = nc.jetstream({ timeout: 1000 }); + await assertRejects( + async () => { + await js.pull("helloworld", "me"); + }, + Error, + ErrorCode.Timeout, + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull consumer doesn't exist", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const { stream } = await initStream(nc); + const js = nc.jetstream({ timeout: 1000 }); + await assertRejects( + async () => { + await js.pull(stream, "me"); + }, + Error, + ErrorCode.Timeout, + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sub - multiple consumers", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + const buf: Promise[] = []; + for (let i = 0; i < 100; i++) { + buf.push(js.publish(subj, Empty)); + } + await Promise.all(buf); + + let ci = await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + assertEquals(ci.num_pending, 100); + + let countA = 0; + let countB = 0; + const m = new Map(); + + const opts = consumerOpts(); + opts.durable("me"); + opts.ackExplicit(); + opts.deliverAll(); + const subA = await js.pullSubscribe(subj, opts); + (async () => { + for await (const msg of subA) { + const v = m.get(msg.seq) ?? 0; + m.set(msg.seq, v + 1); + countA++; + msg.ack(); + } + })().then(); + + const subB = await js.pullSubscribe(subj, opts); + (async () => { + for await (const msg of subB) { + const v = m.get(msg.seq) ?? 0; + m.set(msg.seq, v + 1); + countB++; + msg.ack(); + } + })().then(); + + const done = deferred(); + const interval = setInterval(() => { + if (countA + countB < 100) { + subA.pull({ expires: 500, batch: 25 }); + subB.pull({ expires: 500, batch: 25 }); + } else { + clearInterval(interval); + done.resolve(); + } + }, 25); + + await done; + + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assert(countA > 0); + assert(countB > 0); + assertEquals(countA + countB, 100); + + for (let i = 1; i <= 100; i++) { + assertEquals(m.get(i), 1); + } + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull next", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj); + await js.publish(subj); + + const jsm = await nc.jetstreamManager(); + const si = await jsm.streams.info(stream); + assertEquals(si.state.messages, 2); + + let inbox = ""; + const opts = consumerOpts(); + opts.durable("me"); + opts.ackExplicit(); + opts.manualAck(); + opts.callback((err, msg) => { + if (err) { + if (err.code === ErrorCode.JetStream408RequestTimeout) { + sub.unsubscribe(); + return; + } else { + fail(err.message); + } + } + if (msg) { + msg.next(inbox, { batch: 1, expires: 250 }); + } + }); + const sub = await js.pullSubscribe(subj, opts); + inbox = sub.getSubject(); + sub.pull({ batch: 1, expires: 1000 }); + + await sub.closed; + + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, false); + await sub.closed; + + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 2); + assertEquals(ci.ack_floor.stream_seq, 2); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull errors", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + }); + const js = nc.jetstream(); + + async function expectError( + expires: number, + code: ErrorCode, + ) { + try { + await js.pull(stream, "me", expires); + } catch (err) { + assertEquals(err.code, code); + } + } + + await expectError(0, ErrorCode.JetStream404NoMessages); + await expectError(1000, ErrorCode.JetStream408RequestTimeout); + + await js.publish(subj); + + // we expect a message + const a = await js.pull(stream, "me", 1000); + assertEquals(a.seq, 1); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull error: max_waiting", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.8.2")) { + return; + } + + const { stream } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + max_waiting: 1, + }); + const js = nc.jetstream(); + + async function expectError( + expires: number, + code: ErrorCode, + ): Promise { + const d = deferred(); + try { + await js.pull(stream, "me", expires); + } catch (err) { + d.resolve(err); + assertEquals(err.code, code); + } + return d; + } + await Promise.all([ + expectError( + 3000, + ErrorCode.JetStream408RequestTimeout, + ), + expectError(3000, ErrorCode.JetStream409), + ]); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull error: js not enabled", async () => { + const { ns, nc } = await setup(); + const js = nc.jetstream(); + async function expectError(code: ErrorCode, expires: number) { + const noMsgs = deferred(); + try { + await js.pull("stream", "me", expires); + } catch (err) { + noMsgs.resolve(err); + } + const ne = await noMsgs; + assertEquals(ne.code, code); + } + + await expectError(ErrorCode.JetStreamNotEnabled, 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - ephemeral pull consumer", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const d = deferred(); + const js = nc.jetstream(); + + // no durable name specified + const opts = consumerOpts(); + opts.manualAck(); + opts.ackExplicit(); + opts.deliverAll(); + opts.inactiveEphemeralThreshold(500); + opts.callback((_err, msg) => { + assert(msg !== null); + d.resolve(msg); + }); + + const sub = await js.pullSubscribe(subj, opts); + const old = await sub.consumerInfo(); + + const sc = StringCodec(); + await js.publish(subj, sc.encode("hello")); + sub.pull({ batch: 1, expires: 1000 }); + + const m = await d; + assertEquals(sc.decode(m.data), "hello"); + + sub.unsubscribe(); + await nc.flush(); + + const jsm = await nc.jetstreamManager(); + await delay(1500); + await assertRejects( + async () => { + await jsm.consumers.info(stream, old.name); + }, + Error, + "consumer not found", + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull consumer max_bytes rejected on old servers", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + // change the version of the server to fail pull with max bytes + const nci = nc as NatsConnectionImpl; + nci.features.update("2.7.0"); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + filter_subject: ">", + }); + const js = nc.jetstream() as JetStreamClientImpl; + + const d = deferred(); + + const opts = consumerOpts(); + opts.deliverAll(); + opts.ackExplicit(); + opts.manualAck(); + opts.callback((err, _msg) => { + if (err) { + d.resolve(err); + } + }); + + const sub = await js.pullSubscribe(subj, opts); + assertThrows( + () => { + sub.pull({ expires: 2000, max_bytes: 2 }); + }, + Error, + "max_bytes is only supported on servers 2.8.3 or better", + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull on stopped server doesn't close client", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + (async () => { + let reconnects = 0; + for await (const s of nc.status()) { + switch (s.type) { + case DebugEvents.Reconnecting: + reconnects++; + if (reconnects === 2) { + ns.restart().then((s) => { + ns = s; + }); + } + break; + case Events.Reconnect: + setTimeout(() => { + loop = false; + }); + break; + default: + // nothing + } + } + })().then(); + const jsm = await nc.jetstreamManager(); + const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); + const { name: stream } = si.config; + await jsm.consumers.add(stream, { + durable_name: "dur", + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + setTimeout(() => { + ns.stop(); + }, 2000); + + let loop = true; + let requestTimeouts = 0; + while (true) { + try { + await js.pull(stream, "dur", 500); + } catch (err) { + switch (err.code) { + case ErrorCode.Timeout: + // js is not ready + continue; + case ErrorCode.JetStream408RequestTimeout: + requestTimeouts++; + break; + default: + fail(`unexpected error: ${err.message}`); + break; + } + } + if (!loop) { + break; + } + } + assert(requestTimeouts > 0); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull heartbeat", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + + const reconnected = deferred(); + (async () => { + for await (const s of nc.status()) { + if (s.type === Events.Reconnect) { + // if we reconnect, close the client + reconnected.resolve(); + } + } + })().then(); + + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); + + const js = nc.jetstream(); + + const d = deferred(); + const opts = consumerOpts().ackExplicit().callback((err, m) => { + if (err?.code === ErrorCode.JetStreamIdleHeartBeat) { + d.resolve(); + } + if (m) { + m.ack(); + } + }); + const psub = await js.pullSubscribe("test", opts); + await ns.stop(); + + psub.pull({ idle_heartbeat: 500, expires: 5000, batch: 1 }); + await d; + + ns = await ns.restart(); + // this here because otherwise get a resource leak error in the test + await reconnected; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull heartbeat iter", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + + const reconnected = deferred(); + (async () => { + for await (const s of nc.status()) { + if (s.type === Events.Reconnect) { + // if we reconnect, close the client + reconnected.resolve(); + } + } + })().then(); + + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); + + const js = nc.jetstream(); + + const opts = consumerOpts().ackExplicit(); + const psub = await js.pullSubscribe("test", opts); + const done = assertRejects( + async () => { + for await (const m of psub) { + m.ack(); + } + }, + Error, + "idle heartbeats missed", + ); + + await ns.stop(); + psub.pull({ idle_heartbeat: 500, expires: 5000, batch: 1 }); + await done; + + ns = await ns.restart(); + // this here because otherwise get a resource leak error in the test + await reconnected; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull multi-subject filter", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + if (await notCompatible(ns, nc, "2.10.0")) { + return; + } + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + await jsm.streams.add({ name, subjects: [`a.>`] }); + + const opts = consumerOpts() + .durable("me") + .ackExplicit() + .filterSubject("a.b") + .filterSubject("a.c") + .callback((_err, msg) => { + msg?.ack(); + }); + + const sub = await js.pullSubscribe("a.>", opts); + const ci = await sub.consumerInfo(); + assertExists(ci.config.filter_subjects); + assertArrayIncludes(ci.config.filter_subjects, ["a.b", "a.c"]); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull consumer deleted", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const d = deferred(); + const js = nc.jetstream(); + + js.pull(name, name, 5000) + .catch((err) => { + d.resolve(err); + }); + await nc.flush(); + await jsm.consumers.delete(name, name); + + const err = await d; + assertEquals(err?.message, "consumer deleted"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pullSub cb consumer deleted", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const d = deferred(); + const js = nc.jetstream(); + + const opts = consumerOpts().bind(name, name).callback((err, _m) => { + if (err) { + d.resolve(err); + } + }); + const sub = await js.pullSubscribe(name, opts); + sub.pull({ expires: 5000 }); + await nc.flush(); + await jsm.consumers.delete(name, name); + + const err = await d; + assertEquals(err?.message, "consumer deleted"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pullSub iter consumer deleted", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const d = deferred(); + const js = nc.jetstream(); + + const opts = consumerOpts().bind(name, name); + + const sub = await js.pullSubscribe(name, opts); + (async () => { + for await (const _m of sub) { + // nothing + } + })().catch((err) => { + d.resolve(err); + }); + sub.pull({ expires: 5000 }); + await nc.flush(); + await jsm.consumers.delete(name, name); + + const err = await d; + assertEquals(err?.message, "consumer deleted"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull sync", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + }); + + const js = nc.jetstream(); + + await js.publish(name); + await js.publish(name); + + const sub = await js.pullSubscribe(name, consumerOpts().bind(name, name)); + sub.pull({ batch: 2, no_wait: true }); + const sync = syncIterator(sub); + + assertExists(await sync.next()); + assertExists(await sync.next()); + // if don't unsubscribe, the call will hang because + // we are waiting for the sub.pull() to happen + sub.unsubscribe(); + assertEquals(await sync.next(), null); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - pull single filter", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + if (await notCompatible(ns, nc, "2.10.0")) { + return; + } + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + await jsm.streams.add({ name, subjects: [`a.>`] }); + + const opts = consumerOpts() + .durable("me") + .ackExplicit() + .filterSubject("a.b") + .callback((_err, msg) => { + msg?.ack(); + }); + + const sub = await js.pullSubscribe("a.>", opts); + const ci = await sub.consumerInfo(); + assertEquals(ci.config.filter_subject, "a.b"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - last of", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const jsm = await nc.jetstreamManager(); + const n = nuid.next(); + await jsm.streams.add({ + name: n, + subjects: [`${n}.>`], + }); + + const subja = `${n}.A`; + const subjb = `${n}.B`; + + const js = nc.jetstream(); + + await js.publish(subja, Empty); + await js.publish(subjb, Empty); + await js.publish(subjb, Empty); + await js.publish(subja, Empty); + + const opts = { + durable_name: "B", + filter_subject: subjb, + deliver_policy: DeliverPolicy.Last, + ack_policy: AckPolicy.Explicit, + } as Partial; + + await jsm.consumers.add(n, opts); + const m = await js.pull(n, "B"); + assertEquals(m.seq, 3); + + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/jetstream_pushconsumer_test.ts b/jetstream/tests/jetstream_pushconsumer_test.ts new file mode 100644 index 00000000..93eb6fdc --- /dev/null +++ b/jetstream/tests/jetstream_pushconsumer_test.ts @@ -0,0 +1,1432 @@ +/* + * Copyright 2021-2023 The NATS Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + assertBetween, + cleanup, + jetstreamExportServerConf, + jetstreamServerConf, + notCompatible, + setup, +} from "../../tests/helpers/mod.ts"; +import { initStream } from "./jstest_util.ts"; +import { + createInbox, + DebugEvents, + ErrorCode, + Events, + NatsError, + syncIterator, +} from "../../nats-base-client/core.ts"; +import { + ConsumerOpts, + consumerOpts, + ConsumerOptsBuilderImpl, + JetStreamSubscriptionInfoable, + JsHeaders, + PubAck, +} from "../types.ts"; +import { assertEquals } from "https://deno.land/std@0.200.0/assert/assert_equals.ts"; +import { callbackConsume } from "./jetstream_test.ts"; +import { assertRejects } from "https://deno.land/std@0.200.0/assert/assert_rejects.ts"; +import { + AckPolicy, + DeliverPolicy, + RetentionPolicy, + StorageType, +} from "../jsapi_types.ts"; +import { JSONCodec, StringCodec } from "../../nats-base-client/codec.ts"; +import { assert } from "../../nats-base-client/denobuffer.ts"; +import { Empty } from "../../nats-base-client/encoders.ts"; +import { deferred, delay } from "../../nats-base-client/util.ts"; +import { nuid } from "../../nats-base-client/nuid.ts"; +import { JsMsg } from "../jsmsg.ts"; +import { connect } from "../../src/connect.ts"; +import { + isFlowControlMsg, + isHeartbeatMsg, + Js409Errors, + nanos, +} from "../jsutil.ts"; +import { JetStreamSubscriptionImpl } from "../jsclient.ts"; +import { assertIsError } from "https://deno.land/std@0.200.0/assert/assert_is_error.ts"; +import { assertExists } from "https://deno.land/std@0.200.0/assert/assert_exists.ts"; +import { assertArrayIncludes } from "https://deno.land/std@0.200.0/assert/assert_array_includes.ts"; + +Deno.test("jetstream - ephemeral push", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + const js = nc.jetstream(); + await js.publish(subj); + + const opts = { + max: 1, + config: { deliver_subject: createInbox() }, + } as ConsumerOpts; + opts.callbackFn = callbackConsume(); + const sub = await js.subscribe(subj, opts); + await sub.closed; + assertEquals(sub.getProcessed(), 1); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - durable", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + await js.publish(subj); + + const opts = consumerOpts(); + opts.durable("me"); + opts.manualAck(); + opts.ackExplicit(); + opts.maxMessages(1); + opts.deliverTo(createInbox()); + + let sub = await js.subscribe(subj, opts); + const done = await (async () => { + for await (const m of sub) { + m.ack(); + } + })(); + + await done; + assertEquals(sub.getProcessed(), 1); + + // consumer should exist + const jsm = await nc.jetstreamManager(); + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.name, "me"); + + // delete the consumer + sub = await js.subscribe(subj, opts); + await sub.destroy(); + await assertRejects( + async () => { + await jsm.consumers.info(stream, "me"); + }, + Error, + "consumer not found", + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - queue error checks", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.3.5")) { + return; + } + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + + await assertRejects( + async () => { + const opts = consumerOpts(); + opts.durable("me"); + opts.deliverTo("x"); + opts.queue("x"); + opts.idleHeartbeat(1000); + + await js.subscribe(subj, opts); + }, + Error, + "jetstream idle heartbeat is not supported with queue groups", + undefined, + ); + + await assertRejects( + async () => { + const opts = consumerOpts(); + opts.durable("me"); + opts.deliverTo("x"); + opts.queue("x"); + opts.flowControl(); + + await js.subscribe(subj, opts); + }, + Error, + "jetstream flow control is not supported with queue groups", + undefined, + ); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + deliver_group: "x", + ack_policy: AckPolicy.Explicit, + deliver_subject: "x", + }); + + await assertRejects( + async () => { + await js.subscribe(subj, { + stream: stream, + config: { durable_name: "me", deliver_group: "y" }, + }); + }, + Error, + "durable requires queue group 'x'", + undefined, + ); + + await jsm.consumers.add(stream, { + durable_name: "memo", + ack_policy: AckPolicy.Explicit, + deliver_subject: "z", + }); + + await assertRejects( + async () => { + await js.subscribe(subj, { + stream: stream, + config: { durable_name: "memo", deliver_group: "y" }, + }); + }, + Error, + "durable requires no queue group", + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - max ack pending", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + const sc = StringCodec(); + const d = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]; + const buf: Promise[] = []; + const js = nc.jetstream(); + d.forEach((v) => { + buf.push(js.publish(subj, sc.encode(v), { msgID: v })); + }); + await Promise.all(buf); + + const consumers = await jsm.consumers.list(stream).next(); + assert(consumers.length === 0); + + const opts = consumerOpts(); + opts.maxAckPending(2); + opts.maxMessages(10); + opts.manualAck(); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + await (async () => { + for await (const m of sub) { + assert( + sub.getPending() < 3, + `didn't expect pending messages greater than 2`, + ); + m.ack(); + } + })(); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - subscribe - not attached callback", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.durable("me"); + opts.ackExplicit(); + opts.callback(callbackConsume(false)); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, false); + + await delay(500); + sub.unsubscribe(); + await nc.flush(); + + const jsm = await nc.jetstreamManager(); + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 5); + assertEquals(ci.ack_floor.stream_seq, 5); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - subscribe - not attached non-durable", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.callback(callbackConsume()); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const subin = sub as unknown as JetStreamSubscriptionInfoable; + assert(subin.info); + assertEquals(subin.info.attached, false); + await delay(500); + assertEquals(sub.getProcessed(), 5); + sub.unsubscribe(); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - autoack", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const jsm = await nc.jetstreamManager(); + + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.Explicit, + deliver_subject: createInbox(), + }); + + const js = nc.jetstream(); + await js.publish(subj); + + let ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 1); + + const sopts = consumerOpts(); + sopts.ackAll(); + sopts.durable("me"); + sopts.callback(() => { + // nothing + }); + sopts.maxMessages(1); + const sub = await js.subscribe(subj, sopts); + await sub.closed; + + await nc.flush(); + ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.num_waiting, 0); + assertEquals(ci.num_ack_pending, 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - subscribe - info", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.callback(callbackConsume()); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + await delay(250); + const ci = await sub.consumerInfo(); + assertEquals(ci.delivered.stream_seq, 5); + assertEquals(ci.ack_floor.stream_seq, 5); + await sub.destroy(); + + await assertRejects( + async () => { + await sub.consumerInfo(); + }, + Error, + "consumer not found", + undefined, + ); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - deliver new", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.deliverNew(); + opts.maxMessages(1); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const done = (async () => { + for await (const m of sub) { + assertEquals(m.seq, 6); + } + })(); + await js.publish(subj, Empty, { expect: { lastSequence: 5 } }); + await done; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - deliver last", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.deliverLast(); + opts.maxMessages(1); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const done = (async () => { + for await (const m of sub) { + assertEquals(m.seq, 5); + } + })(); + await done; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - deliver seq", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.startSequence(2); + opts.maxMessages(1); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const done = (async () => { + for await (const m of sub) { + assertEquals(m.seq, 2); + } + })(); + await done; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - deliver start time", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); + await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); + + await delay(1000); + const now = new Date(); + await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.startTime(now); + opts.maxMessages(1); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const done = (async () => { + for await (const m of sub) { + assertEquals(m.seq, 3); + } + })(); + await done; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - deliver last per subject", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc)) { + return; + } + const jsm = await nc.jetstreamManager(); + const stream = nuid.next(); + const subj = `${stream}.*`; + await jsm.streams.add( + { name: stream, subjects: [subj] }, + ); + + const js = nc.jetstream(); + await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 0 } }); + await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 1 } }); + await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 2 } }); + await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 3 } }); + await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 4 } }); + await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 5 } }); + + const opts = consumerOpts(); + opts.ackExplicit(); + opts.deliverLastPerSubject(); + opts.deliverTo(createInbox()); + + const sub = await js.subscribe(subj, opts); + const ci = await sub.consumerInfo(); + const buf: JsMsg[] = []; + assertEquals(ci.num_ack_pending, 2); + const done = (async () => { + for await (const m of sub) { + buf.push(m); + if (buf.length === 2) { + sub.unsubscribe(); + } + } + })(); + await done; + assertEquals(buf[0].info.streamSequence, 5); + assertEquals(buf[1].info.streamSequence, 6); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - cross account subscribe", async () => { + const { ns, nc: admin } = await setup( + jetstreamExportServerConf(), + { + user: "js", + pass: "js", + }, + ); + + // add a stream + const { subj } = await initStream(admin); + const adminjs = admin.jetstream(); + await adminjs.publish(subj); + await adminjs.publish(subj); + + // create a durable config + const bo = consumerOpts() as ConsumerOptsBuilderImpl; + bo.durable("me"); + bo.manualAck(); + bo.maxMessages(2); + bo.deliverTo(createInbox("A")); + + const nc = await connect({ + port: ns.port, + user: "a", + pass: "s3cret", + }); + const js = nc.jetstream({ apiPrefix: "IPA" }); + + const opts = bo.getOpts(); + const acks: Promise[] = []; + const d = deferred(); + const sub = await js.subscribe(subj, opts); + await (async () => { + for await (const m of sub) { + acks.push(m.ackAck()); + if (m.seq === 2) { + d.resolve(); + } + } + })(); + await d; + await Promise.all(acks); + const ci = await sub.consumerInfo(); + assertEquals(ci.num_pending, 0); + assertEquals(ci.delivered.stream_seq, 2); + assertEquals(ci.ack_floor.stream_seq, 2); + await sub.destroy(); + await assertRejects( + async () => { + await sub.consumerInfo(); + }, + Error, + "consumer not found", + undefined, + ); + + await cleanup(ns, admin, nc); +}); + +Deno.test("jetstream - ack lease extends with working", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + + const sn = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: sn, subjects: [`${sn}.>`] }); + + const js = nc.jetstream(); + await js.publish(`${sn}.A`, Empty, { msgID: "1" }); + + const inbox = createInbox(); + const cc = { + "ack_wait": nanos(2000), + "deliver_subject": inbox, + "ack_policy": AckPolicy.Explicit, + "durable_name": "me", + }; + await jsm.consumers.add(sn, cc); + + const opts = consumerOpts(); + opts.durable("me"); + opts.manualAck(); + + const sub = await js.subscribe(`${sn}.>`, opts); + const done = (async () => { + for await (const m of sub) { + const timer = setInterval(() => { + m.working(); + }, 750); + // we got a message now we are going to delay for 31 sec + await delay(15); + const ci = await jsm.consumers.info(sn, "me"); + assertEquals(ci.num_ack_pending, 1); + m.ack(); + clearInterval(timer); + break; + } + })(); + + await done; + + // make sure the message went out + await nc.flush(); + const ci2 = await jsm.consumers.info(sn, "me"); + assertEquals(ci2.delivered.stream_seq, 1); + assertEquals(ci2.num_redelivered, 0); + assertEquals(ci2.num_ack_pending, 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - qsub", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.3.5")) { + return; + } + const { subj } = await initStream(nc); + const js = nc.jetstream(); + + const opts = consumerOpts(); + opts.queue("q"); + opts.durable("n"); + opts.deliverTo("here"); + opts.callback((_err, m) => { + if (m) { + m.ack(); + } + }); + + const sub = await js.subscribe(subj, opts); + const sub2 = await js.subscribe(subj, opts); + + for (let i = 0; i < 100; i++) { + await js.publish(subj, Empty); + } + await nc.flush(); + await sub.drain(); + await sub2.drain(); + + assert(sub.getProcessed() > 0); + assert(sub2.getProcessed() > 0); + assertEquals(sub.getProcessed() + sub2.getProcessed(), 100); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - qsub ackall", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.3.5")) { + return; + } + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + + const opts = consumerOpts(); + opts.queue("q"); + opts.durable("n"); + opts.deliverTo("here"); + opts.ackAll(); + opts.callback((_err, _m) => {}); + + const sub = await js.subscribe(subj, opts); + const sub2 = await js.subscribe(subj, opts); + + for (let i = 0; i < 100; i++) { + await js.publish(subj, Empty); + } + await nc.flush(); + await sub.drain(); + await sub2.drain(); + + assert(sub.getProcessed() > 0); + assert(sub2.getProcessed() > 0); + assertEquals(sub.getProcessed() + sub2.getProcessed(), 100); + + const jsm = await nc.jetstreamManager(); + const ci = await jsm.consumers.info(stream, "n"); + assertEquals(ci.num_pending, 0); + assertEquals(ci.num_ack_pending, 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - idle heartbeats", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + await js.publish(subj); + const jsm = await nc.jetstreamManager(); + const inbox = createInbox(); + await jsm.consumers.add(stream, { + durable_name: "me", + deliver_subject: inbox, + idle_heartbeat: nanos(2000), + }); + + const sub = nc.subscribe(inbox, { + callback: (_err, msg) => { + if (isHeartbeatMsg(msg)) { + assertEquals(msg.headers?.get(JsHeaders.LastConsumerSeqHdr), "1"); + assertEquals(msg.headers?.get(JsHeaders.LastStreamSeqHdr), "1"); + sub.drain(); + } + }, + }); + + await sub.closed; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - flow control", async () => { + const { ns, nc } = await setup(jetstreamServerConf({ + jetstream: { + max_file: -1, + }, + }, true)); + const { stream, subj } = await initStream(nc); + const data = new Uint8Array(1024 * 100); + const js = nc.jetstream(); + const proms = []; + for (let i = 0; i < 2000; i++) { + proms.push(js.publish(subj, data)); + nc.publish(subj, data); + if (proms.length % 100 === 0) { + await Promise.all(proms); + proms.length = 0; + } + } + if (proms.length) { + await Promise.all(proms); + } + await nc.flush(); + + const jsm = await nc.jetstreamManager(); + const inbox = createInbox(); + await jsm.consumers.add(stream, { + durable_name: "me", + deliver_subject: inbox, + flow_control: true, + idle_heartbeat: nanos(750), + }); + + const fc = deferred(); + const hb = deferred(); + const sub = nc.subscribe(inbox, { + callback: (_err, msg) => { + msg.respond(); + if (isFlowControlMsg(msg)) { + fc.resolve(); + } + if (isHeartbeatMsg(msg)) { + hb.resolve(); + } + }, + }); + + await Promise.all([fc, hb]); + sub.unsubscribe(); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - durable resumes", async () => { + let { ns, nc } = await setup(jetstreamServerConf({}, true), { + maxReconnectAttempts: -1, + reconnectTimeWait: 100, + }); + + const { stream, subj } = await initStream(nc); + const jc = JSONCodec(); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + let values = ["a", "b", "c"]; + for (const v of values) { + await js.publish(subj, jc.encode(v)); + } + + const dsubj = createInbox(); + const opts = consumerOpts(); + opts.ackExplicit(); + opts.deliverAll(); + opts.deliverTo(dsubj); + opts.durable("me"); + const sub = await js.subscribe(subj, opts); + const done = (async () => { + for await (const m of sub) { + m.ack(); + if (m.seq === 6) { + sub.unsubscribe(); + } + } + })(); + await nc.flush(); + await ns.stop(); + ns = await ns.restart(); + await delay(300); + values = ["d", "e", "f"]; + for (const v of values) { + await js.publish(subj, jc.encode(v)); + } + await nc.flush(); + await done; + + const si = await jsm.streams.info(stream); + assertEquals(si.state.messages, 6); + const ci = await jsm.consumers.info(stream, "me"); + assertEquals(ci.delivered.stream_seq, 6); + assertEquals(ci.num_pending, 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - reuse consumer", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const id = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + subjects: [`${id}.*`], + name: id, + retention: RetentionPolicy.Workqueue, + }); + + await jsm.consumers.add(id, { + "durable_name": "X", + "deliver_subject": "out", + "deliver_policy": DeliverPolicy.All, + "ack_policy": AckPolicy.Explicit, + "deliver_group": "queuea", + }); + + // second create should be OK, since it is idempotent + await jsm.consumers.add(id, { + "durable_name": "X", + "deliver_subject": "out", + "deliver_policy": DeliverPolicy.All, + "ack_policy": AckPolicy.Explicit, + "deliver_group": "queuea", + }); + + const js = nc.jetstream(); + const opts = consumerOpts(); + opts.ackExplicit(); + opts.durable("X"); + opts.deliverAll(); + opts.deliverTo("out2"); + opts.queue("queuea"); + + const sub = await js.subscribe(`${id}.*`, opts); + const ci = await sub.consumerInfo(); + // the deliver subject we specified should be ignored + // the one specified by the consumer is used + assertEquals(ci.config.deliver_subject, "out"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - nak delay", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.7.1")) { + return; + } + + const { subj } = await initStream(nc); + const js = nc.jetstream(); + await js.publish(subj); + + let start = 0; + + const opts = consumerOpts(); + opts.ackAll(); + opts.deliverTo(createInbox()); + opts.maxMessages(2); + opts.callback((_, m) => { + assert(m); + if (m.redelivered) { + m.ack(); + } else { + start = Date.now(); + m.nak(2000); + } + }); + + const sub = await js.subscribe(subj, opts); + await sub.closed; + + const delay = Date.now() - start; + assertBetween(delay, 1800, 2200); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - redelivery property works", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + if (await notCompatible(ns, nc, "2.3.5")) { + return; + } + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + + let r = 0; + + const opts = consumerOpts(); + opts.ackAll(); + opts.queue("q"); + opts.durable("n"); + opts.deliverTo(createInbox()); + opts.callback((_err, m) => { + if (m) { + if (m.info.redelivered) { + r++; + } + if (m.seq === 100) { + m.ack(); + } + if (m.seq % 3 === 0) { + m.nak(); + } + } + }); + + const sub = await js.subscribe(subj, opts); + const sub2 = await js.subscribe(subj, opts); + + for (let i = 0; i < 100; i++) { + await js.publish(subj, Empty); + } + await nc.flush(); + await sub.drain(); + await sub2.drain(); + + assert(sub.getProcessed() > 0); + assert(sub2.getProcessed() > 0); + assert(r > 0); + assert(sub.getProcessed() + sub2.getProcessed() > 100); + + await nc.flush(); + const jsm = await nc.jetstreamManager(); + const ci = await jsm.consumers.info(stream, "n"); + assert(ci.delivered.consumer_seq > 100); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - bind", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const js = nc.jetstream(); + const jsm = await nc.jetstreamManager(); + const stream = nuid.next(); + const subj = `${stream}.*`; + await jsm.streams.add({ + name: stream, + subjects: [subj], + }); + + const inbox = createInbox(); + await jsm.consumers.add(stream, { + durable_name: "me", + ack_policy: AckPolicy.None, + deliver_subject: inbox, + }); + + const opts = consumerOpts(); + opts.bind(stream, "hello"); + opts.deliverTo(inbox); + + await assertRejects( + async () => { + await js.subscribe(subj, opts); + }, + Error, + `unable to bind - durable consumer hello doesn't exist in ${stream}`, + undefined, + ); + // the rejection happens and the unsub is scheduled, but it is possible that + // the server didn't process it yet - flush to make sure the unsub was seen + await nc.flush(); + + opts.bind(stream, "me"); + const sub = await js.subscribe(subj, opts); + assertEquals(sub.getProcessed(), 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - bind example", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const js = nc.jetstream(); + const jsm = await nc.jetstreamManager(); + const subj = `A.*`; + await jsm.streams.add({ + name: "A", + subjects: [subj], + }); + + const inbox = createInbox(); + await jsm.consumers.add("A", { + durable_name: "me", + ack_policy: AckPolicy.None, + deliver_subject: inbox, + }); + + const opts = consumerOpts(); + opts.bind("A", "me"); + + const sub = await js.subscribe(subj, opts); + assertEquals(sub.getProcessed(), 0); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push bound", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + await jsm.consumers.add(stream, { + durable_name: "me", + deliver_subject: "here", + }); + + const opts = consumerOpts(); + opts.durable("me"); + opts.manualAck(); + opts.ackExplicit(); + opts.deliverTo("here"); + opts.callback((_err, msg) => { + if (msg) { + msg.ack(); + } + }); + const js = nc.jetstream(); + await js.subscribe(subj, opts); + + const nc2 = await connect({ port: ns.port }); + const js2 = nc2.jetstream(); + await assertRejects( + async () => { + await js2.subscribe(subj, opts); + }, + Error, + "duplicate subscription", + ); + + await cleanup(ns, nc, nc2); +}); + +Deno.test("jetstream - idleheartbeats errors repeat in callback push sub", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const js = nc.jetstream(); + await js.publish(subj, Empty); + + const buf: NatsError[] = []; + + const d = deferred(); + const fn = (err: NatsError | null, _msg: JsMsg | null): void => { + if (err) { + buf.push(err); + if (buf.length === 3) { + d.resolve(); + } + } + }; + + const opts = consumerOpts(); + opts.durable("me"); + opts.manualAck(); + opts.ackExplicit(); + opts.idleHeartbeat(800); + opts.deliverTo(createInbox()); + opts.callback(fn); + + const sub = await js.subscribe(subj, opts) as JetStreamSubscriptionImpl; + assert(sub.monitor); + await delay(3000); + sub.monitor._change(100, 0, 3); + + buf.forEach((err) => { + assertIsError(err, NatsError, Js409Errors.IdleHeartbeatMissed); + }); + + assertEquals(sub.sub.isClosed(), false); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - idleheartbeats errors in iterator push sub", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { subj } = await initStream(nc); + + const opts = consumerOpts(); + opts.durable("me"); + opts.manualAck(); + opts.ackExplicit(); + opts.idleHeartbeat(800); + opts.deliverTo(createInbox()); + + const js = nc.jetstream(); + const sub = await js.subscribe(subj, opts) as JetStreamSubscriptionImpl; + + const d = deferred(); + (async () => { + for await (const _m of sub) { + // not going to get anything + } + })().catch((err) => { + d.resolve(err); + }); + assert(sub.monitor); + await delay(1700); + sub.monitor._change(100, 0, 1); + const err = await d; + assertIsError(err, NatsError, Js409Errors.IdleHeartbeatMissed); + assertEquals(err.code, ErrorCode.JetStreamIdleHeartBeat); + assertEquals(sub.sub.isClosed(), true); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - bind ephemeral can get consumer info", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + + const jsm = await nc.jetstreamManager(); + + async function testEphemeral(deliverSubject = ""): Promise { + const ci = await jsm.consumers.add(stream, { + ack_policy: AckPolicy.Explicit, + inactive_threshold: nanos(5000), + deliver_subject: deliverSubject, + }); + + const js = nc.jetstream(); + const opts = consumerOpts(); + opts.bind(stream, ci.name); + const sub = deliverSubject + ? await js.subscribe(subj, opts) + : await js.pullSubscribe(subj, opts); + const sci = await sub.consumerInfo(); + assertEquals( + sci.name, + ci.name, + `failed getting ci for ${deliverSubject ? "push" : "pull"}`, + ); + } + + await testEphemeral(); + await testEphemeral(createInbox()); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - create ephemeral with config can get consumer info", async () => { + const { ns, nc } = await setup(jetstreamServerConf({}, true)); + const { stream, subj } = await initStream(nc); + const js = nc.jetstream(); + + async function testEphemeral(deliverSubject = ""): Promise { + const opts = { + stream, + config: { + ack_policy: AckPolicy.Explicit, + deliver_subject: deliverSubject, + }, + }; + const sub = deliverSubject + ? await js.subscribe(subj, opts) + : await js.pullSubscribe(subj, opts); + const ci = await sub.consumerInfo(); + assert( + ci.name, + `failed getting name for ${deliverSubject ? "push" : "pull"}`, + ); + assert( + !ci.config.durable_name, + `unexpected durable name for ${deliverSubject ? "push" : "pull"}`, + ); + } + + await testEphemeral(); + await testEphemeral(createInbox()); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push on stopped server doesn't close client", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + const reconnected = deferred(); + (async () => { + let reconnects = 0; + for await (const s of nc.status()) { + switch (s.type) { + case DebugEvents.Reconnecting: + reconnects++; + if (reconnects === 2) { + ns.restart().then((s) => { + ns = s; + }); + } + break; + case Events.Reconnect: + setTimeout(() => { + reconnected.resolve(); + }, 1000); + break; + default: + // nothing + } + } + })().then(); + const jsm = await nc.jetstreamManager(); + const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); + const { name: stream } = si.config; + + const js = nc.jetstream(); + + await jsm.consumers.add(stream, { + durable_name: "dur", + ack_policy: AckPolicy.Explicit, + deliver_subject: "bar", + }); + + const opts = consumerOpts().manualAck().deliverTo(nuid.next()); + const sub = await js.subscribe("test", opts); + (async () => { + for await (const m of sub) { + m.ack(); + } + })().then(); + + setTimeout(() => { + ns.stop(); + }, 2000); + + await reconnected; + assertEquals(nc.isClosed(), false); + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push heartbeat iter", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + + const reconnected = deferred(); + (async () => { + for await (const s of nc.status()) { + if (s.type === Events.Reconnect) { + // if we reconnect, close the client + reconnected.resolve(); + } + } + })().then(); + + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); + + const js = nc.jetstream(); + + const opts = consumerOpts({ idle_heartbeat: nanos(500) }).ackExplicit() + .deliverTo(nuid.next()); + const psub = await js.subscribe("test", opts); + const done = assertRejects( + async () => { + for await (const m of psub) { + m.ack(); + } + }, + Error, + "idle heartbeats missed", + ); + + await ns.stop(); + await done; + + ns = await ns.restart(); + // this here because otherwise get a resource leak error in the test + await reconnected; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push heartbeat callback", async () => { + let { ns, nc } = await setup(jetstreamServerConf(), { + maxReconnectAttempts: -1, + }); + + const reconnected = deferred(); + (async () => { + for await (const s of nc.status()) { + if (s.type === Events.Reconnect) { + // if we reconnect, close the client + reconnected.resolve(); + } + } + })().then(); + + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); + + const js = nc.jetstream(); + const d = deferred(); + const opts = consumerOpts({ idle_heartbeat: nanos(500) }).ackExplicit() + .deliverTo(nuid.next()) + .callback((err, m) => { + if (err?.code === ErrorCode.JetStreamIdleHeartBeat) { + d.resolve(); + } + if (m) { + m.ack(); + } + }); + await js.subscribe("test", opts); + await ns.stop(); + await d; + + ns = await ns.restart(); + // this here because otherwise get a resource leak error in the test + await reconnected; + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push multi-subject filter", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + if (await notCompatible(ns, nc, "2.10.0")) { + return; + } + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + await jsm.streams.add({ name, subjects: [`a.>`] }); + + const opts = consumerOpts() + .durable("me") + .ackExplicit() + .filterSubject("a.b") + .filterSubject("a.c") + .deliverTo(createInbox()) + .callback((_err, msg) => { + msg?.ack(); + }); + + const sub = await js.subscribe("a.>", opts); + const ci = await sub.consumerInfo(); + assertExists(ci.config.filter_subjects); + assertArrayIncludes(ci.config.filter_subjects, ["a.b", "a.c"]); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push single filter", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + if (await notCompatible(ns, nc, "2.10.0")) { + return; + } + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + const js = nc.jetstream(); + await jsm.streams.add({ name, subjects: [`a.>`] }); + + const opts = consumerOpts() + .durable("me") + .ackExplicit() + .filterSubject("a.b") + .deliverTo(createInbox()) + .callback((_err, msg) => { + msg?.ack(); + }); + + const sub = await js.subscribe("a.>", opts); + const ci = await sub.consumerInfo(); + assertEquals(ci.config.filter_subject, "a.b"); + + await cleanup(ns, nc); +}); + +Deno.test("jetstream - push sync", async () => { + const { ns, nc } = await setup(jetstreamServerConf()); + const name = nuid.next(); + const jsm = await nc.jetstreamManager(); + await jsm.streams.add({ + name, + subjects: [name], + storage: StorageType.Memory, + }); + await jsm.consumers.add(name, { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_subject: "here", + }); + + const js = nc.jetstream(); + + await js.publish(name); + await js.publish(name); + + const sub = await js.subscribe(name, consumerOpts().bind(name, name)); + const sync = syncIterator(sub); + assertExists(await sync.next()); + assertExists(await sync.next()); + + await cleanup(ns, nc); +}); diff --git a/jetstream/tests/jetstream_test.ts b/jetstream/tests/jetstream_test.ts index a485eead..a3361e07 100644 --- a/jetstream/tests/jetstream_test.ts +++ b/jetstream/tests/jetstream_test.ts @@ -85,7 +85,7 @@ import { } from "../types.ts"; import { syncIterator } from "../../nats-base-client/core.ts"; -function callbackConsume(debug = false): JsMsgCallback { +export function callbackConsume(debug = false): JsMsgCallback { return (err: NatsError | null, jm: JsMsg | null) => { if (err) { switch (err.code) { @@ -109,7 +109,7 @@ function callbackConsume(debug = false): JsMsgCallback { }; } -async function consume(iter: QueuedIterator): Promise { +export async function consume(iter: QueuedIterator): Promise { const buf: JsMsg[] = []; await (async () => { for await (const m of iter) { @@ -362,1300 +362,16 @@ Deno.test("jetstream - publish require last sequence by subject", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - ephemeral push", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - const js = nc.jetstream(); - await js.publish(subj); - - const opts = { - max: 1, - config: { deliver_subject: createInbox() }, - } as ConsumerOpts; - opts.callbackFn = callbackConsume(); - const sub = await js.subscribe(subj, opts); - await sub.closed; - assertEquals(sub.getProcessed(), 1); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - durable", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - await js.publish(subj); - - const opts = consumerOpts(); - opts.durable("me"); - opts.manualAck(); - opts.ackExplicit(); - opts.maxMessages(1); - opts.deliverTo(createInbox()); - - let sub = await js.subscribe(subj, opts); - const done = await (async () => { - for await (const m of sub) { - m.ack(); - } - })(); - - await done; - assertEquals(sub.getProcessed(), 1); - - // consumer should exist - const jsm = await nc.jetstreamManager(); - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.name, "me"); - - // delete the consumer - sub = await js.subscribe(subj, opts); - await sub.destroy(); - await assertRejects( - async () => { - await jsm.consumers.info(stream, "me"); - }, - Error, - "consumer not found", - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - queue error checks", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.3.5")) { - return; - } - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - - await assertRejects( - async () => { - const opts = consumerOpts(); - opts.durable("me"); - opts.deliverTo("x"); - opts.queue("x"); - opts.idleHeartbeat(1000); - - await js.subscribe(subj, opts); - }, - Error, - "jetstream idle heartbeat is not supported with queue groups", - undefined, - ); - - await assertRejects( - async () => { - const opts = consumerOpts(); - opts.durable("me"); - opts.deliverTo("x"); - opts.queue("x"); - opts.flowControl(); - - await js.subscribe(subj, opts); - }, - Error, - "jetstream flow control is not supported with queue groups", - undefined, - ); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - deliver_group: "x", - ack_policy: AckPolicy.Explicit, - deliver_subject: "x", - }); - - await assertRejects( - async () => { - await js.subscribe(subj, { - stream: stream, - config: { durable_name: "me", deliver_group: "y" }, - }); - }, - Error, - "durable requires queue group 'x'", - undefined, - ); - - await jsm.consumers.add(stream, { - durable_name: "memo", - ack_policy: AckPolicy.Explicit, - deliver_subject: "z", - }); - - await assertRejects( - async () => { - await js.subscribe(subj, { - stream: stream, - config: { durable_name: "memo", deliver_group: "y" }, - }); - }, - Error, - "durable requires no queue group", - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull no messages", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - await assertRejects( - async () => { - await js.pull(stream, "me"); - }, - Error, - "no messages", - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - await js.publish(subj, Empty, { msgID: "a" }); - let ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - - const jm = await js.pull(stream, "me"); - jm.ack(); - await nc.flush(); - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 1); - assertEquals(ci.ack_floor.stream_seq, 1, JSON.stringify(ci)); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch expires waits", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - const start = Date.now(); - const iter = js.fetch(stream, "me", { expires: 1000 }); - await (async () => { - for await (const _m of iter) { - // nothing - } - })(); - const elapsed = Date.now() - start; - assertBetween(elapsed, 950, 1050); - assertEquals(iter.getReceived(), 0); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch expires waits after initial", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - await js.publish(subj, Empty); - const start = Date.now(); - const iter = js.fetch(stream, "me", { expires: 1000, batch: 5 }); - await (async () => { - for await (const _m of iter) { - // nothing - } - })(); - const elapsed = Date.now() - start; - assertBetween(elapsed, 950, 1050); - assertEquals(iter.getReceived(), 1); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch expires or no_wait is required", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - assertThrows( - () => { - js.fetch(stream, "me"); - }, - Error, - "expires or no_wait is required", - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch: no_wait with more left", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - await js.publish(subj); - await js.publish(subj); - - const iter = js.fetch(stream, "me", { no_wait: true }); - await consume(iter); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch some messages", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - // try to get messages = none available - let sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); - await (async () => { - for await (const m of sub) { - m.ack(); - } - })(); - assertEquals(sub.getProcessed(), 0); - - // seed some messages - await js.publish(subj, Empty, { msgID: "a" }); - await js.publish(subj, Empty, { msgID: "b" }); - await js.publish(subj, Empty, { msgID: "c" }); - - // try to get 2 messages - OK - sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); - await (async () => { - for await (const m of sub) { - m.ack(); - } - })(); - assertEquals(sub.getProcessed(), 2); - - await nc.flush(); - let ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - assertEquals(ci.delivered.stream_seq, 2); - assertEquals(ci.ack_floor.stream_seq, 2); - - // try to get 2 messages - OK, but only gets 1 - sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); - await (async () => { - for await (const m of sub) { - m.ack(); - } - })(); - assertEquals(sub.getProcessed(), 1); - - await nc.flush(); - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 3); - assertEquals(ci.ack_floor.stream_seq, 3); - - // try to get 2 messages - OK, none available - sub = await js.fetch(stream, "me", { batch: 2, no_wait: true }); - await (async () => { - for await (const m of sub) { - m.ack(); - } - })(); - assertEquals(sub.getProcessed(), 0); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - max ack pending", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - const sc = StringCodec(); - const d = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]; - const buf: Promise[] = []; - const js = nc.jetstream(); - d.forEach((v) => { - buf.push(js.publish(subj, sc.encode(v), { msgID: v })); - }); - await Promise.all(buf); - - const consumers = await jsm.consumers.list(stream).next(); - assert(consumers.length === 0); - - const opts = consumerOpts(); - opts.maxAckPending(2); - opts.maxMessages(10); - opts.manualAck(); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - await (async () => { - for await (const m of sub) { - assert( - sub.getPending() < 3, - `didn't expect pending messages greater than 2`, - ); - m.ack(); - } - })(); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - ephemeral options", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - const v = await jsm.consumers.add(stream, { - inactive_threshold: nanos(1000), - ack_policy: AckPolicy.Explicit, - }); - assertEquals(v.config.inactive_threshold, nanos(1000)); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull consumer options", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - const v = await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - max_batch: 10, - max_expires: nanos(20000), - }); - - assertEquals(v.config.max_batch, 10); - assertEquals(v.config.max_expires, nanos(20000)); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub - attached iterator", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const jc = JSONCodec(); - - let sum = 0; - const opts = consumerOpts(); - opts.durable("me"); - - const js = nc.jetstream(); - const sub = await js.pullSubscribe(subj, opts); - (async () => { - for await (const msg of sub) { - assert(msg); - //@ts-ignore: test - const ne = checkJsError(msg.msg); - if (ne) { - console.log(ne.message); - } - const n = jc.decode(msg.data); - sum += n; - msg.ack(); - } - })().then(); - sub.pull({ expires: 500, batch: 5 }); - - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, true); - await delay(250); - assertEquals(sum, 0); - - let ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 0); - assertEquals(ci.ack_floor.stream_seq, 0); - - await js.publish(subj, jc.encode(1), { msgID: "1" }); - await js.publish(subj, jc.encode(2), { msgID: "2" }); - sub.pull({ expires: 500, batch: 5 }); - await delay(500); - assertEquals(sum, 3); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 2); - assertEquals(ci.ack_floor.stream_seq, 2); - - await js.publish(subj, jc.encode(3), { msgID: "3" }); - await js.publish(subj, jc.encode(5), { msgID: "4" }); - sub.pull({ expires: 500, batch: 5 }); - await delay(1000); - assertEquals(sum, 11); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 4); - assertEquals(ci.ack_floor.stream_seq, 4); - - await js.publish(subj, jc.encode(7), { msgID: "5" }); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub - attached callback", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const jc = JSONCodec(); - - let sum = 0; - const opts = consumerOpts(); - opts.durable("me"); - - opts.callback((err, msg) => { - if (err) { - switch (err.code) { - case ErrorCode.JetStream408RequestTimeout: - case ErrorCode.JetStream409: - case ErrorCode.JetStream404NoMessages: - return; - default: - fail(err.code); - } - } - if (msg) { - const n = jc.decode(msg.data); - sum += n; - msg.ack(); - } - }); - - const js = nc.jetstream(); - const sub = await js.pullSubscribe(subj, opts); - sub.pull({ expires: 500, batch: 5 }); - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, true); - await delay(250); - assertEquals(sum, 0); - - let ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 0); - assertEquals(ci.ack_floor.stream_seq, 0); - - await js.publish(subj, jc.encode(1), { msgID: "1" }); - await js.publish(subj, jc.encode(2), { msgID: "2" }); - sub.pull({ expires: 500, batch: 5 }); - await delay(500); - assertEquals(sum, 3); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 2); - assertEquals(ci.ack_floor.stream_seq, 2); - - await js.publish(subj, jc.encode(3), { msgID: "3" }); - await js.publish(subj, jc.encode(5), { msgID: "4" }); - sub.pull({ expires: 500, batch: 5 }); - await delay(1000); - assertEquals(sum, 11); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 4); - assertEquals(ci.ack_floor.stream_seq, 4); - - await js.publish(subj, jc.encode(7), { msgID: "5" }); - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub - not attached callback", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj); - - const opts = consumerOpts(); - opts.durable("me"); - opts.ackExplicit(); - opts.maxMessages(1); - opts.callback(callbackConsume(false)); - - const sub = await js.pullSubscribe(subj, opts); - sub.pull(); - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, false); - await sub.closed; - - const jsm = await nc.jetstreamManager(); - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 1); - assertEquals(ci.ack_floor.stream_seq, 1); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub requires explicit", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - - await assertRejects( - async () => { - const opts = consumerOpts(); - opts.durable("me"); - opts.ackAll(); - await js.pullSubscribe(subj, opts); - }, - Error, - "ack policy for pull", - undefined, - ); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub ephemeral", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj); - - const d = deferred(); - const opts = consumerOpts(); - - opts.ackExplicit(); - opts.callback((err, msg) => { - if (err) { - d.reject(err); - } else { - d.resolve(msg!); - } - }); - - const ps = await js.pullSubscribe(subj, opts); - ps.pull({ no_wait: true }); - const r = await d; - assertEquals(r.subject, subj); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - subscribe - not attached callback", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.durable("me"); - opts.ackExplicit(); - opts.callback(callbackConsume(false)); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, false); - - await delay(500); - sub.unsubscribe(); - await nc.flush(); - - const jsm = await nc.jetstreamManager(); - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 5); - assertEquals(ci.ack_floor.stream_seq, 5); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - subscribe - not attached non-durable", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.callback(callbackConsume()); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, false); - await delay(500); - assertEquals(sub.getProcessed(), 5); - sub.unsubscribe(); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch none - breaks after expires", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const sw = time(); - const batch = js.fetch(stream, "me", { - batch: 10, - expires: 1000, - }); - const done = (async () => { - for await (const m of batch) { - console.log(m.info); - fail("expected no messages"); - } - })(); - - await done; - sw.mark(); - sw.assertInRange(1000); - assertEquals(batch.getReceived(), 0); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch none - no wait breaks fast", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const sw = time(); - const batch = js.fetch(stream, "me", { - batch: 10, - no_wait: true, - }); - const done = (async () => { - for await (const m of batch) { - m.ack(); - } - })(); - - await done; - sw.mark(); - assertBetween(sw.duration(), 0, 500); - assertEquals(batch.getReceived(), 0); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch one - no wait breaks fast", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - await js.publish(subj); - - const sw = time(); - const batch = js.fetch(stream, "me", { - batch: 10, - no_wait: true, - }); - const done = (async () => { - for await (const m of batch) { - m.ack(); - } - })(); - - await done; - sw.mark(); - console.log({ duration: sw.duration() }); - const duration = sw.duration(); - assert(150 > duration, `${duration}`); - assertEquals(batch.getReceived(), 1); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch none - cancel timers", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const sw = time(); - const batch = js.fetch(stream, "me", { - batch: 10, - expires: 1000, - }); - const done = (async () => { - for await (const m of batch) { - m.ack(); - } - })(); - - const nci = nc as NatsConnectionImpl; - const last = nci.protocol.subscriptions.sidCounter; - const sub = nci.protocol.subscriptions.get(last); - assert(sub); - sub.unsubscribe(); - - await done; - sw.mark(); - assert(25 > sw.duration()); - assertEquals(batch.getReceived(), 0); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch one - breaks after expires", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - await js.publish(subj); - - const sw = time(); - const batch = js.fetch(stream, "me", { - batch: 10, - expires: 1000, - }); - const done = (async () => { - for await (const m of batch) { - m.ack(); - } - })(); - - await done; - sw.mark(); - sw.assertInRange(1000); - assertEquals(batch.getReceived(), 1); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull consumer info without pull", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - await js.publish(subj); - - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - - const sopts = consumerOpts(); - sopts.durable("me"); - await assertRejects( - async () => { - await js.subscribe(subj, sopts); - }, - Error, - "push consumer requires deliver_subject", - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - autoack", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - deliver_subject: createInbox(), - }); - - const js = nc.jetstream(); - await js.publish(subj); - - let ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 1); - - const sopts = consumerOpts(); - sopts.ackAll(); - sopts.durable("me"); - sopts.callback(() => { - // nothing - }); - sopts.maxMessages(1); - const sub = await js.subscribe(subj, sopts); - await sub.closed; - - await nc.flush(); - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.num_waiting, 0); - assertEquals(ci.num_ack_pending, 0); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - subscribe - info", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.callback(callbackConsume()); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - await delay(250); - const ci = await sub.consumerInfo(); - assertEquals(ci.delivered.stream_seq, 5); - assertEquals(ci.ack_floor.stream_seq, 5); - await sub.destroy(); - - await assertRejects( - async () => { - await sub.consumerInfo(); - }, - Error, - "consumer not found", - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - deliver new", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.deliverNew(); - opts.maxMessages(1); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const done = (async () => { - for await (const m of sub) { - assertEquals(m.seq, 6); - } - })(); - await js.publish(subj, Empty, { expect: { lastSequence: 5 } }); - await done; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - deliver last", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.deliverLast(); - opts.maxMessages(1); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const done = (async () => { - for await (const m of sub) { - assertEquals(m.seq, 5); - } - })(); - await done; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - last of", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const jsm = await nc.jetstreamManager(); - const n = nuid.next(); - await jsm.streams.add({ - name: n, - subjects: [`${n}.>`], - }); - - const subja = `${n}.A`; - const subjb = `${n}.B`; - - const js = nc.jetstream(); - - await js.publish(subja, Empty); - await js.publish(subjb, Empty); - await js.publish(subjb, Empty); - await js.publish(subja, Empty); - - const opts = { - durable_name: "B", - filter_subject: subjb, - deliver_policy: DeliverPolicy.Last, - ack_policy: AckPolicy.Explicit, - } as Partial; - - await jsm.consumers.add(n, opts); - const m = await js.pull(n, "B"); - assertEquals(m.seq, 3); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - deliver seq", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 3 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 4 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.startSequence(2); - opts.maxMessages(1); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const done = (async () => { - for await (const m of sub) { - assertEquals(m.seq, 2); - } - })(); - await done; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - deliver start time", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty, { expect: { lastSequence: 0 } }); - await js.publish(subj, Empty, { expect: { lastSequence: 1 } }); - - await delay(1000); - const now = new Date(); - await js.publish(subj, Empty, { expect: { lastSequence: 2 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.startTime(now); - opts.maxMessages(1); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const done = (async () => { - for await (const m of sub) { - assertEquals(m.seq, 3); - } - })(); - await done; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - deliver last per subject", async () => { +Deno.test("jetstream - ephemeral options", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc)) { - return; - } + const { stream } = await initStream(nc); const jsm = await nc.jetstreamManager(); - const stream = nuid.next(); - const subj = `${stream}.*`; - await jsm.streams.add( - { name: stream, subjects: [subj] }, - ); - - const js = nc.jetstream(); - await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 0 } }); - await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 1 } }); - await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 2 } }); - await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 3 } }); - await js.publish(`${stream}.A`, Empty, { expect: { lastSequence: 4 } }); - await js.publish(`${stream}.B`, Empty, { expect: { lastSequence: 5 } }); - - const opts = consumerOpts(); - opts.ackExplicit(); - opts.deliverLastPerSubject(); - opts.deliverTo(createInbox()); - - const sub = await js.subscribe(subj, opts); - const ci = await sub.consumerInfo(); - const buf: JsMsg[] = []; - assertEquals(ci.num_ack_pending, 2); - const done = (async () => { - for await (const m of sub) { - buf.push(m); - if (buf.length === 2) { - sub.unsubscribe(); - } - } - })(); - await done; - assertEquals(buf[0].info.streamSequence, 5); - assertEquals(buf[1].info.streamSequence, 6); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - cross account subscribe", async () => { - const { ns, nc: admin } = await setup( - jetstreamExportServerConf(), - { - user: "js", - pass: "js", - }, - ); - - // add a stream - const { subj } = await initStream(admin); - const adminjs = admin.jetstream(); - await adminjs.publish(subj); - await adminjs.publish(subj); - - // create a durable config - const bo = consumerOpts() as ConsumerOptsBuilderImpl; - bo.durable("me"); - bo.manualAck(); - bo.maxMessages(2); - bo.deliverTo(createInbox("A")); - - const nc = await connect({ - port: ns.port, - user: "a", - pass: "s3cret", - }); - const js = nc.jetstream({ apiPrefix: "IPA" }); - - const opts = bo.getOpts(); - const acks: Promise[] = []; - const d = deferred(); - const sub = await js.subscribe(subj, opts); - await (async () => { - for await (const m of sub) { - acks.push(m.ackAck()); - if (m.seq === 2) { - d.resolve(); - } - } - })(); - await d; - await Promise.all(acks); - const ci = await sub.consumerInfo(); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 2); - assertEquals(ci.ack_floor.stream_seq, 2); - await sub.destroy(); - await assertRejects( - async () => { - await sub.consumerInfo(); - }, - Error, - "consumer not found", - undefined, - ); - - await cleanup(ns, admin, nc); -}); - -Deno.test("jetstream - cross account pull subscribe", () => { - disabled("cross account pull subscribe test needs updating"); - // const { ns, nc: admin } = await setup( - // jetstreamExportServerConf(), - // { - // user: "js", - // pass: "js", - // }, - // ); - // - // // add a stream - // const { stream, subj } = await initStream(admin); - // const adminjs = admin.jetstream(); - // await adminjs.publish(subj); - // await adminjs.publish(subj); - // - // // FIXME: create a durable config - // const bo = consumerOpts() as ConsumerOptsBuilderImpl; - // bo.manualAck(); - // bo.ackExplicit(); - // bo.maxMessages(2); - // bo.durable("me"); - // - // // pull subscriber stalls - // const nc = await connect({ - // port: ns.port, - // user: "a", - // pass: "s3cret", - // inboxPrefix: "A", - // }); - // const js = nc.jetstream({ apiPrefix: "IPA" }); - // - // const opts = bo.getOpts(); - // const sub = await js.pullSubscribe(subj, opts); - // const done = (async () => { - // for await (const m of sub) { - // m.ack(); - // } - // })(); - // sub.pull({ batch: 2 }); - // await done; - // assertEquals(sub.getProcessed(), 2); - // - // const ci = await sub.consumerInfo(); - // assertEquals(ci.num_pending, 0); - // assertEquals(ci.delivered.stream_seq, 2); - // - // await sub.destroy(); - // await assertThrowsAsync( - // async () => { - // await sub.consumerInfo(); - // }, - // Error, - // "consumer not found", - // ); - // - // await cleanup(ns, admin, nc); -}); - -Deno.test("jetstream - cross account pull", async () => { - const { ns, nc: admin } = await setup( - jetstreamExportServerConf(), - { - user: "js", - pass: "js", - }, - ); - - // add a stream - const { stream, subj } = await initStream(admin); - const admjs = admin.jetstream(); - await admjs.publish(subj); - await admjs.publish(subj); - - const admjsm = await admin.jetstreamManager(); - - // create a durable config - const bo = consumerOpts() as ConsumerOptsBuilderImpl; - bo.manualAck(); - bo.ackExplicit(); - bo.durable("me"); - const opts = bo.getOpts(); - await admjsm.consumers.add(stream, opts.config); - - const nc = await connect({ - port: ns.port, - user: "a", - pass: "s3cret", - inboxPrefix: "A", + const v = await jsm.consumers.add(stream, { + inactive_threshold: nanos(1000), + ack_policy: AckPolicy.Explicit, }); - - // the api prefix is not used for pull/fetch() - const js = nc.jetstream({ apiPrefix: "IPA" }); - let msg = await js.pull(stream, "me"); - assertEquals(msg.seq, 1); - msg = await js.pull(stream, "me"); - assertEquals(msg.seq, 2); - await assertRejects( - async () => { - await js.pull(stream, "me"); - }, - Error, - "no messages", - undefined, - ); - - await cleanup(ns, admin, nc); + assertEquals(v.config.inactive_threshold, nanos(1000)); + await cleanup(ns, nc); }); Deno.test("jetstream - publish headers", async () => { @@ -1677,154 +393,6 @@ Deno.test("jetstream - publish headers", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - pull stream doesn't exist", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const js = nc.jetstream({ timeout: 1000 }); - await assertRejects( - async () => { - await js.pull("helloworld", "me"); - }, - Error, - ErrorCode.Timeout, - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull consumer doesn't exist", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream } = await initStream(nc); - const js = nc.jetstream({ timeout: 1000 }); - await assertRejects( - async () => { - await js.pull(stream, "me"); - }, - Error, - ErrorCode.Timeout, - undefined, - ); - - await cleanup(ns, nc); -}); - -// Deno.test("jetstream - cross account fetch", async () => { -// const { ns, nc: admin } = await setup( -// jetstreamExportServerConf(), -// { -// user: "js", -// pass: "js", -// }, -// ); -// -// // add a stream -// const { stream, subj } = await initStream(admin); -// const admjs = admin.jetstream(); -// await admjs.publish(subj, Empty, {msgID: "1"}); -// await admjs.publish(subj, Empty, {msgID: "2"}); -// -// const admjsm = await admin.jetstreamManager(); -// -// // create a durable config -// const bo = consumerOpts() as ConsumerOptsBuilderImpl; -// bo.manualAck(); -// bo.ackExplicit(); -// bo.durable("me"); -// bo.maxAckPending(10); -// const opts = bo.getOpts(); -// await admjsm.consumers.add(stream, opts.config); -// -// const nc = await connect({ -// port: ns.port, -// user: "a", -// pass: "s3cret", -// inboxPrefix: "A", -// debug: true, -// }); -// -// // the api prefix is not used for pull/fetch() -// const js = nc.jetstream({ apiPrefix: "IPA" }); -// let iter = js.fetch(stream, "me", { batch: 20, expires: 1000 }); -// const msgs = await consume(iter); -// -// assertEquals(msgs.length, 2); -// -// // msg = await js.pull(stream, "me"); -// // assertEquals(msg.seq, 2); -// // await assertThrowsAsync(async () => { -// // await js.pull(stream, "me"); -// // }, Error, "No Messages"); -// -// await cleanup(ns, admin, nc); -// }); - -Deno.test("jetstream - pull consumer doesn't exist", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const { stream } = await initStream(nc); - const js = nc.jetstream({ timeout: 1000 }); - await assertRejects( - async () => { - await js.pull(stream, "me"); - }, - Error, - ErrorCode.Timeout, - undefined, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - ack lease extends with working", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - - const sn = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: sn, subjects: [`${sn}.>`] }); - - const js = nc.jetstream(); - await js.publish(`${sn}.A`, Empty, { msgID: "1" }); - - const inbox = createInbox(); - const cc = { - "ack_wait": nanos(2000), - "deliver_subject": inbox, - "ack_policy": AckPolicy.Explicit, - "durable_name": "me", - }; - await jsm.consumers.add(sn, cc); - - const opts = consumerOpts(); - opts.durable("me"); - opts.manualAck(); - - const sub = await js.subscribe(`${sn}.>`, opts); - const done = (async () => { - for await (const m of sub) { - const timer = setInterval(() => { - m.working(); - }, 750); - // we got a message now we are going to delay for 31 sec - await delay(15); - const ci = await jsm.consumers.info(sn, "me"); - assertEquals(ci.num_ack_pending, 1); - m.ack(); - clearInterval(timer); - break; - } - })(); - - await done; - - // make sure the message went out - await nc.flush(); - const ci2 = await jsm.consumers.info(sn, "me"); - assertEquals(ci2.delivered.stream_seq, 1); - assertEquals(ci2.num_redelivered, 0); - assertEquals(ci2.num_ack_pending, 0); - - await cleanup(ns, nc); -}); - Deno.test("jetstream - JSON", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); const { stream, subj } = await initStream(nc); @@ -1854,157 +422,6 @@ Deno.test("jetstream - JSON", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - qsub", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.3.5")) { - return; - } - const { subj } = await initStream(nc); - const js = nc.jetstream(); - - const opts = consumerOpts(); - opts.queue("q"); - opts.durable("n"); - opts.deliverTo("here"); - opts.callback((_err, m) => { - if (m) { - m.ack(); - } - }); - - const sub = await js.subscribe(subj, opts); - const sub2 = await js.subscribe(subj, opts); - - for (let i = 0; i < 100; i++) { - await js.publish(subj, Empty); - } - await nc.flush(); - await sub.drain(); - await sub2.drain(); - - assert(sub.getProcessed() > 0); - assert(sub2.getProcessed() > 0); - assertEquals(sub.getProcessed() + sub2.getProcessed(), 100); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - qsub ackall", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.3.5")) { - return; - } - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - - const opts = consumerOpts(); - opts.queue("q"); - opts.durable("n"); - opts.deliverTo("here"); - opts.ackAll(); - opts.callback((_err, _m) => {}); - - const sub = await js.subscribe(subj, opts); - const sub2 = await js.subscribe(subj, opts); - - for (let i = 0; i < 100; i++) { - await js.publish(subj, Empty); - } - await nc.flush(); - await sub.drain(); - await sub2.drain(); - - assert(sub.getProcessed() > 0); - assert(sub2.getProcessed() > 0); - assertEquals(sub.getProcessed() + sub2.getProcessed(), 100); - - const jsm = await nc.jetstreamManager(); - const ci = await jsm.consumers.info(stream, "n"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.num_ack_pending, 0); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - idle heartbeats", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - await js.publish(subj); - const jsm = await nc.jetstreamManager(); - const inbox = createInbox(); - await jsm.consumers.add(stream, { - durable_name: "me", - deliver_subject: inbox, - idle_heartbeat: nanos(2000), - }); - - const sub = nc.subscribe(inbox, { - callback: (_err, msg) => { - if (isHeartbeatMsg(msg)) { - assertEquals(msg.headers?.get(JsHeaders.LastConsumerSeqHdr), "1"); - assertEquals(msg.headers?.get(JsHeaders.LastStreamSeqHdr), "1"); - sub.drain(); - } - }, - }); - - await sub.closed; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - flow control", async () => { - const { ns, nc } = await setup(jetstreamServerConf({ - jetstream: { - max_file: -1, - }, - }, true)); - const { stream, subj } = await initStream(nc); - const data = new Uint8Array(1024 * 100); - const js = nc.jetstream(); - const proms = []; - for (let i = 0; i < 2000; i++) { - proms.push(js.publish(subj, data)); - nc.publish(subj, data); - if (proms.length % 100 === 0) { - await Promise.all(proms); - proms.length = 0; - } - } - if (proms.length) { - await Promise.all(proms); - } - await nc.flush(); - - const jsm = await nc.jetstreamManager(); - const inbox = createInbox(); - await jsm.consumers.add(stream, { - durable_name: "me", - deliver_subject: inbox, - flow_control: true, - idle_heartbeat: nanos(750), - }); - - const fc = deferred(); - const hb = deferred(); - const sub = nc.subscribe(inbox, { - callback: (_err, msg) => { - msg.respond(); - if (isFlowControlMsg(msg)) { - fc.resolve(); - } - if (isHeartbeatMsg(msg)) { - hb.resolve(); - } - }, - }); - - await Promise.all([fc, hb]); - sub.unsubscribe(); - await cleanup(ns, nc); -}); - Deno.test("jetstream - domain", async () => { const { ns, nc } = await setup( jetstreamServerConf({ @@ -2031,69 +448,19 @@ Deno.test("jetstream - account domain", async () => { A: { users: [ { user: "a", password: "a" }, - ], - jetstream: { max_memory: 10000, max_file: 10000 }, - }, - }, - }, true); - - const { ns, nc } = await setup(conf, { user: "a", pass: "a" }); - - const jsm = await nc.jetstreamManager({ domain: "A" }); - const ai = await jsm.getAccountInfo(); - assert(ai.domain, "A"); - //@ts-ignore: internal use - assertEquals(jsm.prefix, `$JS.A.API`); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - durable resumes", async () => { - let { ns, nc } = await setup(jetstreamServerConf({}, true), { - maxReconnectAttempts: -1, - reconnectTimeWait: 100, - }); - - const { stream, subj } = await initStream(nc); - const jc = JSONCodec(); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - let values = ["a", "b", "c"]; - for (const v of values) { - await js.publish(subj, jc.encode(v)); - } - - const dsubj = createInbox(); - const opts = consumerOpts(); - opts.ackExplicit(); - opts.deliverAll(); - opts.deliverTo(dsubj); - opts.durable("me"); - const sub = await js.subscribe(subj, opts); - const done = (async () => { - for await (const m of sub) { - m.ack(); - if (m.seq === 6) { - sub.unsubscribe(); - } - } - })(); - await nc.flush(); - await ns.stop(); - ns = await ns.restart(); - await delay(300); - values = ["d", "e", "f"]; - for (const v of values) { - await js.publish(subj, jc.encode(v)); - } - await nc.flush(); - await done; + ], + jetstream: { max_memory: 10000, max_file: 10000 }, + }, + }, + }, true); - const si = await jsm.streams.info(stream); - assertEquals(si.state.messages, 6); - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.delivered.stream_seq, 6); - assertEquals(ci.num_pending, 0); + const { ns, nc } = await setup(conf, { user: "a", pass: "a" }); + const jsm = await nc.jetstreamManager({ domain: "A" }); + const ai = await jsm.getAccountInfo(); + assert(ai.domain, "A"); + //@ts-ignore: internal use + assertEquals(jsm.prefix, `$JS.A.API`); await cleanup(ns, nc); }); @@ -2148,121 +515,6 @@ Deno.test("jetstream - cleanup", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - reuse consumer", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const id = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - subjects: [`${id}.*`], - name: id, - retention: RetentionPolicy.Workqueue, - }); - - await jsm.consumers.add(id, { - "durable_name": "X", - "deliver_subject": "out", - "deliver_policy": DeliverPolicy.All, - "ack_policy": AckPolicy.Explicit, - "deliver_group": "queuea", - }); - - // second create should be OK, since it is idempotent - await jsm.consumers.add(id, { - "durable_name": "X", - "deliver_subject": "out", - "deliver_policy": DeliverPolicy.All, - "ack_policy": AckPolicy.Explicit, - "deliver_group": "queuea", - }); - - const js = nc.jetstream(); - const opts = consumerOpts(); - opts.ackExplicit(); - opts.durable("X"); - opts.deliverAll(); - opts.deliverTo("out2"); - opts.queue("queuea"); - - const sub = await js.subscribe(`${id}.*`, opts); - const ci = await sub.consumerInfo(); - // the deliver subject we specified should be ignored - // the one specified by the consumer is used - assertEquals(ci.config.deliver_subject, "out"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sub - multiple consumers", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - const buf: Promise[] = []; - for (let i = 0; i < 100; i++) { - buf.push(js.publish(subj, Empty)); - } - await Promise.all(buf); - - let ci = await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - assertEquals(ci.num_pending, 100); - - let countA = 0; - let countB = 0; - const m = new Map(); - - const opts = consumerOpts(); - opts.durable("me"); - opts.ackExplicit(); - opts.deliverAll(); - const subA = await js.pullSubscribe(subj, opts); - (async () => { - for await (const msg of subA) { - const v = m.get(msg.seq) ?? 0; - m.set(msg.seq, v + 1); - countA++; - msg.ack(); - } - })().then(); - - const subB = await js.pullSubscribe(subj, opts); - (async () => { - for await (const msg of subB) { - const v = m.get(msg.seq) ?? 0; - m.set(msg.seq, v + 1); - countB++; - msg.ack(); - } - })().then(); - - const done = deferred(); - const interval = setInterval(() => { - if (countA + countB < 100) { - subA.pull({ expires: 500, batch: 25 }); - subB.pull({ expires: 500, batch: 25 }); - } else { - clearInterval(interval); - done.resolve(); - } - }, 25); - - await done; - - ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assert(countA > 0); - assert(countB > 0); - assertEquals(countA + countB, 100); - - for (let i = 1; i <= 100; i++) { - assertEquals(m.get(i), 1); - } - - await cleanup(ns, nc); -}); - Deno.test("jetstream - source", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); @@ -2317,91 +569,6 @@ Deno.test("jetstream - source", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - nak delay", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.7.1")) { - return; - } - - const { subj } = await initStream(nc); - const js = nc.jetstream(); - await js.publish(subj); - - let start = 0; - - const opts = consumerOpts(); - opts.ackAll(); - opts.deliverTo(createInbox()); - opts.maxMessages(2); - opts.callback((_, m) => { - assert(m); - if (m.redelivered) { - m.ack(); - } else { - start = Date.now(); - m.nak(2000); - } - }); - - const sub = await js.subscribe(subj, opts); - await sub.closed; - - const delay = Date.now() - start; - assertBetween(delay, 1800, 2200); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - redelivery property works", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.3.5")) { - return; - } - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - - let r = 0; - - const opts = consumerOpts(); - opts.ackAll(); - opts.queue("q"); - opts.durable("n"); - opts.deliverTo(createInbox()); - opts.callback((_err, m) => { - if (m) { - if (m.info.redelivered) { - r++; - } - if (m.seq === 100) { - m.ack(); - } - if (m.seq % 3 === 0) { - m.nak(); - } - } - }); - - const sub = await js.subscribe(subj, opts); - const sub2 = await js.subscribe(subj, opts); - - for (let i = 0; i < 100; i++) { - await js.publish(subj, Empty); - } - await nc.flush(); - await sub.drain(); - await sub2.drain(); - - assert(sub.getProcessed() > 0); - assert(sub2.getProcessed() > 0); - assert(r > 0); - assert(sub.getProcessed() + sub2.getProcessed() > 100); - - await nc.flush(); - const jsm = await nc.jetstreamManager(); - const ci = await jsm.consumers.info(stream, "n"); - assert(ci.delivered.consumer_seq > 100); - await cleanup(ns, nc); -}); - async function ocTest( N: number, S: number, @@ -2871,47 +1038,6 @@ Deno.test("jetstream - can access kv", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - bind", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const js = nc.jetstream(); - const jsm = await nc.jetstreamManager(); - const stream = nuid.next(); - const subj = `${stream}.*`; - await jsm.streams.add({ - name: stream, - subjects: [subj], - }); - - const inbox = createInbox(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.None, - deliver_subject: inbox, - }); - - const opts = consumerOpts(); - opts.bind(stream, "hello"); - opts.deliverTo(inbox); - - await assertRejects( - async () => { - await js.subscribe(subj, opts); - }, - Error, - `unable to bind - durable consumer hello doesn't exist in ${stream}`, - undefined, - ); - // the rejection happens and the unsub is scheduled, but it is possible that - // the server didn't process it yet - flush to make sure the unsub was seen - await nc.flush(); - - opts.bind(stream, "me"); - const sub = await js.subscribe(subj, opts); - assertEquals(sub.getProcessed(), 0); - - await cleanup(ns, nc); -}); - Deno.test("jetstream - bind with diff subject fails", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); const jsm = await nc.jetstreamManager(); @@ -2944,32 +1070,6 @@ Deno.test("jetstream - bind with diff subject fails", async () => { await cleanup(ns, nc); }); -Deno.test("jetstream - bind example", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const js = nc.jetstream(); - const jsm = await nc.jetstreamManager(); - const subj = `A.*`; - await jsm.streams.add({ - name: "A", - subjects: [subj], - }); - - const inbox = createInbox(); - await jsm.consumers.add("A", { - durable_name: "me", - ack_policy: AckPolicy.None, - deliver_subject: inbox, - }); - - const opts = consumerOpts(); - opts.bind("A", "me"); - - const sub = await js.subscribe(subj, opts); - assertEquals(sub.getProcessed(), 0); - - await cleanup(ns, nc); -}); - Deno.test("jetstream - test events stream", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); const js = nc.jetstream(); @@ -2988,181 +1088,39 @@ Deno.test("jetstream - test events stream", async () => { durable_name: "me", filter_subject: "events.>", }, - callbackFn: (_err: NatsError | null, msg: JsMsg | null) => { - msg?.ack(); - }, - }); - - await js.publish("events.a"); - await js.publish("events.b"); - await delay(2000); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - bind without consumer should fail", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const js = nc.jetstream(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name: "events", - subjects: ["events.>"], - }); - - const opts = consumerOpts(); - opts.manualAck(); - opts.ackExplicit(); - opts.bind("events", "hello"); - - await assertRejects( - async () => { - await js.subscribe("events.>", opts); - }, - Error, - "unable to bind - durable consumer hello doesn't exist in events", - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull next", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj); - await js.publish(subj); - - const jsm = await nc.jetstreamManager(); - const si = await jsm.streams.info(stream); - assertEquals(si.state.messages, 2); - - let inbox = ""; - const opts = consumerOpts(); - opts.durable("me"); - opts.ackExplicit(); - opts.manualAck(); - opts.callback((err, msg) => { - if (err) { - if (err.code === ErrorCode.JetStream408RequestTimeout) { - sub.unsubscribe(); - return; - } else { - fail(err.message); - } - } - if (msg) { - msg.next(inbox, { batch: 1, expires: 250 }); - } - }); - const sub = await js.pullSubscribe(subj, opts); - inbox = sub.getSubject(); - sub.pull({ batch: 1, expires: 1000 }); - - await sub.closed; - - const subin = sub as unknown as JetStreamSubscriptionInfoable; - assert(subin.info); - assertEquals(subin.info.attached, false); - await sub.closed; - - const ci = await jsm.consumers.info(stream, "me"); - assertEquals(ci.num_pending, 0); - assertEquals(ci.delivered.stream_seq, 2); - assertEquals(ci.ack_floor.stream_seq, 2); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull errors", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - const js = nc.jetstream(); - - async function expectError( - expires: number, - code: ErrorCode, - ) { - try { - await js.pull(stream, "me", expires); - } catch (err) { - assertEquals(err.code, code); - } - } - - await expectError(0, ErrorCode.JetStream404NoMessages); - await expectError(1000, ErrorCode.JetStream408RequestTimeout); - - await js.publish(subj); - - // we expect a message - const a = await js.pull(stream, "me", 1000); - assertEquals(a.seq, 1); + callbackFn: (_err: NatsError | null, msg: JsMsg | null) => { + msg?.ack(); + }, + }); + await js.publish("events.a"); + await js.publish("events.b"); + await delay(2000); await cleanup(ns, nc); }); -Deno.test("jetstream - pull error: max_waiting", async () => { +Deno.test("jetstream - bind without consumer should fail", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); - if (await notCompatible(ns, nc, "2.8.2")) { - return; - } - - const { stream } = await initStream(nc); + const js = nc.jetstream(); const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - max_waiting: 1, + await jsm.streams.add({ + name: "events", + subjects: ["events.>"], }); - const js = nc.jetstream(); - - async function expectError( - expires: number, - code: ErrorCode, - ): Promise { - const d = deferred(); - try { - await js.pull(stream, "me", expires); - } catch (err) { - d.resolve(err); - assertEquals(err.code, code); - } - return d; - } - await Promise.all([ - expectError( - 3000, - ErrorCode.JetStream408RequestTimeout, - ), - expectError(3000, ErrorCode.JetStream409), - ]); - await cleanup(ns, nc); -}); + const opts = consumerOpts(); + opts.manualAck(); + opts.ackExplicit(); + opts.bind("events", "hello"); -Deno.test("jetstream - pull error: js not enabled", async () => { - const { ns, nc } = await setup(); - const js = nc.jetstream(); - async function expectError(code: ErrorCode, expires: number) { - const noMsgs = deferred(); - try { - await js.pull("stream", "me", expires); - } catch (err) { - noMsgs.resolve(err); - } - const ne = await noMsgs; - assertEquals(ne.code, code); - } + await assertRejects( + async () => { + await js.subscribe("events.>", opts); + }, + Error, + "unable to bind - durable consumer hello doesn't exist in events", + ); - await expectError(ErrorCode.JetStreamNotEnabled, 0); await cleanup(ns, nc); }); @@ -3222,376 +1180,59 @@ Deno.test("jetstream - backoff", async () => { const js = nc.jetstream(); await js.publish(subj); - const opts = consumerOpts(); - opts.bind(stream, "me"); - opts.manualAck(); - - const arrive: number[] = []; - let start = 0; - const sub = await js.subscribe(subj, opts); - await (async () => { - for await (const m of sub) { - if (start === 0) { - start = Date.now(); - } - arrive.push(Date.now()); - if (m.info.redeliveryCount === 4) { - break; - } - } - })(); - - const delta = arrive.map((v) => { - return v - start; - }); - - assert(delta[1] > 250 && delta[1] < 1000); - assert(delta[2] > 1250 && delta[2] < 1500); - assert(delta[3] > 4250 && delta[2] < 4500); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push bound", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - deliver_subject: "here", - }); - - const opts = consumerOpts(); - opts.durable("me"); - opts.manualAck(); - opts.ackExplicit(); - opts.deliverTo("here"); - opts.callback((_err, msg) => { - if (msg) { - msg.ack(); - } - }); - const js = nc.jetstream(); - await js.subscribe(subj, opts); - - const nc2 = await connect({ port: ns.port }); - const js2 = nc2.jetstream(); - await assertRejects( - async () => { - await js2.subscribe(subj, opts); - }, - Error, - "duplicate subscription", - ); - - await cleanup(ns, nc, nc2); -}); - -Deno.test("jetstream - detailed errors", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const jsm = await nc.jetstreamManager(); - - const ne = await assertRejects(() => { - return jsm.streams.add({ - name: "test", - num_replicas: 3, - subjects: ["foo"], - }); - }) as NatsError; - - assert(ne.api_error); - assertEquals( - ne.message, - "replicas > 1 not supported in non-clustered mode", - ); - assertEquals( - ne.api_error.description, - "replicas > 1 not supported in non-clustered mode", - ); - assertEquals(ne.api_error.code, 500); - assertEquals(ne.api_error.err_code, 10074); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - ephemeral pull consumer", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - const d = deferred(); - const js = nc.jetstream(); - - // no durable name specified - const opts = consumerOpts(); - opts.manualAck(); - opts.ackExplicit(); - opts.deliverAll(); - opts.inactiveEphemeralThreshold(500); - opts.callback((_err, msg) => { - assert(msg !== null); - d.resolve(msg); - }); - - const sub = await js.pullSubscribe(subj, opts); - const old = await sub.consumerInfo(); - - const sc = StringCodec(); - await js.publish(subj, sc.encode("hello")); - sub.pull({ batch: 1, expires: 1000 }); - - const m = await d; - assertEquals(sc.decode(m.data), "hello"); - - sub.unsubscribe(); - await nc.flush(); - - const jsm = await nc.jetstreamManager(); - await delay(1500); - await assertRejects( - async () => { - await jsm.consumers.info(stream, old.name); - }, - Error, - "consumer not found", - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull consumer max_bytes rejected on old servers", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - - // change the version of the server to fail pull with max bytes - const nci = nc as NatsConnectionImpl; - nci.features.update("2.7.0"); - - const jsm = await nc.jetstreamManager(); - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - filter_subject: ">", - }); - const js = nc.jetstream() as JetStreamClientImpl; - - const d = deferred(); - - const opts = consumerOpts(); - opts.deliverAll(); - opts.ackExplicit(); - opts.manualAck(); - opts.callback((err, _msg) => { - if (err) { - d.resolve(err); - } - }); - - const sub = await js.pullSubscribe(subj, opts); - assertThrows( - () => { - sub.pull({ expires: 2000, max_bytes: 2 }); - }, - Error, - "max_bytes is only supported on servers 2.8.3 or better", - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - idleheartbeat missed on fetch", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const iter = js.fetch(stream, "me", { - expires: 2000, - idle_heartbeat: 250, - //@ts-ignore: testing - delay_heartbeat: true, - }); - - await assertRejects( - async () => { - for await (const _m of iter) { - // no message expected - } - }, - NatsError, - Js409Errors.IdleHeartbeatMissed, - ); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - idleheartbeat on fetch", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - - await jsm.consumers.add(stream, { - durable_name: "me", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - const iter = js.fetch(stream, "me", { - expires: 2000, - idle_heartbeat: 250, - }); - - // we don't expect this to throw - await (async () => { - for await (const _m of iter) { - // no message expected - } - })(); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - idleheartbeats errors repeat in callback push sub", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const js = nc.jetstream(); - await js.publish(subj, Empty); - - const buf: NatsError[] = []; - - const d = deferred(); - const fn = (err: NatsError | null, _msg: JsMsg | null): void => { - if (err) { - buf.push(err); - if (buf.length === 3) { - d.resolve(); - } - } - }; - - const opts = consumerOpts(); - opts.durable("me"); - opts.manualAck(); - opts.ackExplicit(); - opts.idleHeartbeat(800); - opts.deliverTo(createInbox()); - opts.callback(fn); - - const sub = await js.subscribe(subj, opts) as JetStreamSubscriptionImpl; - assert(sub.monitor); - await delay(3000); - sub.monitor._change(100, 0, 3); - - buf.forEach((err) => { - assertIsError(err, NatsError, Js409Errors.IdleHeartbeatMissed); - }); - - assertEquals(sub.sub.isClosed(), false); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - idleheartbeats errors in iterator push sub", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { subj } = await initStream(nc); - - const opts = consumerOpts(); - opts.durable("me"); - opts.manualAck(); - opts.ackExplicit(); - opts.idleHeartbeat(800); - opts.deliverTo(createInbox()); - - const js = nc.jetstream(); - const sub = await js.subscribe(subj, opts) as JetStreamSubscriptionImpl; + const opts = consumerOpts(); + opts.bind(stream, "me"); + opts.manualAck(); - const d = deferred(); - (async () => { - for await (const _m of sub) { - // not going to get anything + const arrive: number[] = []; + let start = 0; + const sub = await js.subscribe(subj, opts); + await (async () => { + for await (const m of sub) { + if (start === 0) { + start = Date.now(); + } + arrive.push(Date.now()); + if (m.info.redeliveryCount === 4) { + break; + } } - })().catch((err) => { - d.resolve(err); + })(); + + const delta = arrive.map((v) => { + return v - start; }); - assert(sub.monitor); - await delay(1700); - sub.monitor._change(100, 0, 1); - const err = await d; - assertIsError(err, NatsError, Js409Errors.IdleHeartbeatMissed); - assertEquals(err.code, ErrorCode.JetStreamIdleHeartBeat); - assertEquals(sub.sub.isClosed(), true); + + assert(delta[1] > 250 && delta[1] < 1000); + assert(delta[2] > 1250 && delta[2] < 1500); + assert(delta[3] > 4250 && delta[2] < 4500); await cleanup(ns, nc); }); -Deno.test("jetstream - bind ephemeral can get consumer info", async () => { +Deno.test("jetstream - detailed errors", async () => { const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const jsm = await nc.jetstreamManager(); - async function testEphemeral(deliverSubject = ""): Promise { - const ci = await jsm.consumers.add(stream, { - ack_policy: AckPolicy.Explicit, - inactive_threshold: nanos(5000), - deliver_subject: deliverSubject, + const ne = await assertRejects(() => { + return jsm.streams.add({ + name: "test", + num_replicas: 3, + subjects: ["foo"], }); + }) as NatsError; - const js = nc.jetstream(); - const opts = consumerOpts(); - opts.bind(stream, ci.name); - const sub = deliverSubject - ? await js.subscribe(subj, opts) - : await js.pullSubscribe(subj, opts); - const sci = await sub.consumerInfo(); - assertEquals( - sci.name, - ci.name, - `failed getting ci for ${deliverSubject ? "push" : "pull"}`, - ); - } - - await testEphemeral(); - await testEphemeral(createInbox()); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - create ephemeral with config can get consumer info", async () => { - const { ns, nc } = await setup(jetstreamServerConf({}, true)); - const { stream, subj } = await initStream(nc); - const js = nc.jetstream(); - - async function testEphemeral(deliverSubject = ""): Promise { - const opts = { - stream, - config: { - ack_policy: AckPolicy.Explicit, - deliver_subject: deliverSubject, - }, - }; - const sub = deliverSubject - ? await js.subscribe(subj, opts) - : await js.pullSubscribe(subj, opts); - const ci = await sub.consumerInfo(); - assert( - ci.name, - `failed getting name for ${deliverSubject ? "push" : "pull"}`, - ); - assert( - !ci.config.durable_name, - `unexpected durable name for ${deliverSubject ? "push" : "pull"}`, - ); - } - - await testEphemeral(); - await testEphemeral(createInbox()); + assert(ne.api_error); + assertEquals( + ne.message, + "replicas > 1 not supported in non-clustered mode", + ); + assertEquals( + ne.api_error.description, + "replicas > 1 not supported in non-clustered mode", + ); + assertEquals(ne.api_error.code, 500); + assertEquals(ne.api_error.err_code, 10074); await cleanup(ns, nc); }); @@ -3839,423 +1480,39 @@ Deno.test("jetstream - ordered consumer reset", async () => { fail(err.message); } c.unsubscribe(); - d.resolve(m!); - }); - const c = await js.subscribe(subj, opts); - - // stop the server and wait until hbs are missed - await ns.stop(); - while (true) { - const missed = (c as JetStreamSubscriptionImpl).monitor?.missed || 0; - const connected = (nc as NatsConnectionImpl).protocol.connected; - // we want to wait until after 2 because we want to have a cycle - // where we try to recreate the consumer, but skip it because we are - // not connected - if (!connected && missed >= 3) { - break; - } - await delay(300); - } - ns = await ns.restart(); - let ack: PubAck; - while (true) { - try { - ack = await js.publish(subj); - break; - } catch (err) { - if (err.code !== ErrorCode.Timeout) { - fail(err.message); - } - await delay(1000); - } - } - await c.closed; - - assertEquals((await d).seq, ack.seq); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch on stopped server doesn't close client", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - (async () => { - let reconnects = 0; - for await (const s of nc.status()) { - switch (s.type) { - case DebugEvents.Reconnecting: - reconnects++; - if (reconnects === 2) { - ns.restart().then((s) => { - ns = s; - }); - } - break; - case Events.Reconnect: - setTimeout(() => { - loop = false; - }); - break; - default: - // nothing - } - } - })().then(); - const jsm = await nc.jetstreamManager(); - const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); - const { name: stream } = si.config; - await jsm.consumers.add(stream, { - durable_name: "dur", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - - setTimeout(() => { - ns.stop(); - }, 2000); - - let loop = true; - while (true) { - try { - const iter = js.fetch(stream, "dur", { batch: 1, expires: 500 }); - for await (const m of iter) { - m.ack(); - } - if (!loop) { - break; - } - } catch (err) { - fail(`shouldn't have errored: ${err.message}`); - } - } - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull on stopped server doesn't close client", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - (async () => { - let reconnects = 0; - for await (const s of nc.status()) { - switch (s.type) { - case DebugEvents.Reconnecting: - reconnects++; - if (reconnects === 2) { - ns.restart().then((s) => { - ns = s; - }); - } - break; - case Events.Reconnect: - setTimeout(() => { - loop = false; - }); - break; - default: - // nothing - } - } - })().then(); - const jsm = await nc.jetstreamManager(); - const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); - const { name: stream } = si.config; - await jsm.consumers.add(stream, { - durable_name: "dur", - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - setTimeout(() => { - ns.stop(); - }, 2000); - - let loop = true; - let requestTimeouts = 0; - while (true) { - try { - await js.pull(stream, "dur", 500); - } catch (err) { - switch (err.code) { - case ErrorCode.Timeout: - // js is not ready - continue; - case ErrorCode.JetStream408RequestTimeout: - requestTimeouts++; - break; - default: - fail(`unexpected error: ${err.message}`); - break; - } - } - if (!loop) { - break; - } - } - assert(requestTimeouts > 0); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push on stopped server doesn't close client", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - const reconnected = deferred(); - (async () => { - let reconnects = 0; - for await (const s of nc.status()) { - switch (s.type) { - case DebugEvents.Reconnecting: - reconnects++; - if (reconnects === 2) { - ns.restart().then((s) => { - ns = s; - }); - } - break; - case Events.Reconnect: - setTimeout(() => { - reconnected.resolve(); - }, 1000); - break; - default: - // nothing - } - } - })().then(); - const jsm = await nc.jetstreamManager(); - const si = await jsm.streams.add({ name: nuid.next(), subjects: ["test"] }); - const { name: stream } = si.config; - - const js = nc.jetstream(); - - await jsm.consumers.add(stream, { - durable_name: "dur", - ack_policy: AckPolicy.Explicit, - deliver_subject: "bar", - }); - - const opts = consumerOpts().manualAck().deliverTo(nuid.next()); - const sub = await js.subscribe("test", opts); - (async () => { - for await (const m of sub) { - m.ack(); - } - })().then(); - - setTimeout(() => { - ns.stop(); - }, 2000); - - await reconnected; - assertEquals(nc.isClosed(), false); - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch heartbeat", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - - const d = deferred(); - (async () => { - for await (const s of nc.status()) { - if (s.type === Events.Reconnect) { - // if we reconnect, close the client - d.resolve(); - } - } - })().then(); - - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); - const js = nc.jetstream(); - await ns.stop(); - - const iter = js.fetch("my-stream", "dur", { - batch: 1, - expires: 5000, - idle_heartbeat: 500, - }); - - await assertRejects( - async () => { - for await (const m of iter) { - m.ack(); - } - }, - Error, - "idle heartbeats missed", - ); - ns = await ns.restart(); - // this here because otherwise get a resource leak error in the test - await d; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull heartbeat", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - - const reconnected = deferred(); - (async () => { - for await (const s of nc.status()) { - if (s.type === Events.Reconnect) { - // if we reconnect, close the client - reconnected.resolve(); - } - } - })().then(); - - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); - - const js = nc.jetstream(); - - const d = deferred(); - const opts = consumerOpts().ackExplicit().callback((err, m) => { - if (err?.code === ErrorCode.JetStreamIdleHeartBeat) { - d.resolve(); - } - if (m) { - m.ack(); - } - }); - const psub = await js.pullSubscribe("test", opts); - await ns.stop(); - - psub.pull({ idle_heartbeat: 500, expires: 5000, batch: 1 }); - await d; - - ns = await ns.restart(); - // this here because otherwise get a resource leak error in the test - await reconnected; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull heartbeat iter", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - - const reconnected = deferred(); - (async () => { - for await (const s of nc.status()) { - if (s.type === Events.Reconnect) { - // if we reconnect, close the client - reconnected.resolve(); - } - } - })().then(); - - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); - - const js = nc.jetstream(); - - const opts = consumerOpts().ackExplicit(); - const psub = await js.pullSubscribe("test", opts); - const done = assertRejects( - async () => { - for await (const m of psub) { - m.ack(); - } - }, - Error, - "idle heartbeats missed", - ); - - await ns.stop(); - psub.pull({ idle_heartbeat: 500, expires: 5000, batch: 1 }); - await done; - - ns = await ns.restart(); - // this here because otherwise get a resource leak error in the test - await reconnected; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push heartbeat iter", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - - const reconnected = deferred(); - (async () => { - for await (const s of nc.status()) { - if (s.type === Events.Reconnect) { - // if we reconnect, close the client - reconnected.resolve(); - } - } - })().then(); - - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); - - const js = nc.jetstream(); - - const opts = consumerOpts({ idle_heartbeat: nanos(500) }).ackExplicit() - .deliverTo(nuid.next()); - const psub = await js.subscribe("test", opts); - const done = assertRejects( - async () => { - for await (const m of psub) { - m.ack(); - } - }, - Error, - "idle heartbeats missed", - ); + d.resolve(m!); + }); + const c = await js.subscribe(subj, opts); + // stop the server and wait until hbs are missed await ns.stop(); - await done; - + while (true) { + const missed = (c as JetStreamSubscriptionImpl).monitor?.missed || 0; + const connected = (nc as NatsConnectionImpl).protocol.connected; + // we want to wait until after 2 because we want to have a cycle + // where we try to recreate the consumer, but skip it because we are + // not connected + if (!connected && missed >= 3) { + break; + } + await delay(300); + } ns = await ns.restart(); - // this here because otherwise get a resource leak error in the test - await reconnected; - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push heartbeat callback", async () => { - let { ns, nc } = await setup(jetstreamServerConf(), { - maxReconnectAttempts: -1, - }); - - const reconnected = deferred(); - (async () => { - for await (const s of nc.status()) { - if (s.type === Events.Reconnect) { - // if we reconnect, close the client - reconnected.resolve(); + let ack: PubAck; + while (true) { + try { + ack = await js.publish(subj); + break; + } catch (err) { + if (err.code !== ErrorCode.Timeout) { + fail(err.message); } + await delay(1000); } - })().then(); - - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ name: "my-stream", subjects: ["test"] }); - - const js = nc.jetstream(); - const d = deferred(); - const opts = consumerOpts({ idle_heartbeat: nanos(500) }).ackExplicit() - .deliverTo(nuid.next()) - .callback((err, m) => { - if (err?.code === ErrorCode.JetStreamIdleHeartBeat) { - d.resolve(); - } - if (m) { - m.ack(); - } - }); - await js.subscribe("test", opts); - await ns.stop(); - await d; + } + await c.closed; - ns = await ns.restart(); - // this here because otherwise get a resource leak error in the test - await reconnected; + assertEquals((await d).seq, ack.seq); await cleanup(ns, nc); }); @@ -4272,112 +1529,6 @@ Deno.test("jetstream - consumer opt multi subject filter", () => { assertArrayIncludes(co.config.filter_subjects, ["foo", "bar"]); }); -Deno.test("jetstream - push multi-subject filter", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - if (await notCompatible(ns, nc, "2.10.0")) { - return; - } - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - await jsm.streams.add({ name, subjects: [`a.>`] }); - - const opts = consumerOpts() - .durable("me") - .ackExplicit() - .filterSubject("a.b") - .filterSubject("a.c") - .deliverTo(createInbox()) - .callback((_err, msg) => { - msg?.ack(); - }); - - const sub = await js.subscribe("a.>", opts); - const ci = await sub.consumerInfo(); - assertExists(ci.config.filter_subjects); - assertArrayIncludes(ci.config.filter_subjects, ["a.b", "a.c"]); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull multi-subject filter", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - if (await notCompatible(ns, nc, "2.10.0")) { - return; - } - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - await jsm.streams.add({ name, subjects: [`a.>`] }); - - const opts = consumerOpts() - .durable("me") - .ackExplicit() - .filterSubject("a.b") - .filterSubject("a.c") - .callback((_err, msg) => { - msg?.ack(); - }); - - const sub = await js.pullSubscribe("a.>", opts); - const ci = await sub.consumerInfo(); - assertExists(ci.config.filter_subjects); - assertArrayIncludes(ci.config.filter_subjects, ["a.b", "a.c"]); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push single filter", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - if (await notCompatible(ns, nc, "2.10.0")) { - return; - } - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - await jsm.streams.add({ name, subjects: [`a.>`] }); - - const opts = consumerOpts() - .durable("me") - .ackExplicit() - .filterSubject("a.b") - .deliverTo(createInbox()) - .callback((_err, msg) => { - msg?.ack(); - }); - - const sub = await js.subscribe("a.>", opts); - const ci = await sub.consumerInfo(); - assertEquals(ci.config.filter_subject, "a.b"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull single filter", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - if (await notCompatible(ns, nc, "2.10.0")) { - return; - } - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - const js = nc.jetstream(); - await jsm.streams.add({ name, subjects: [`a.>`] }); - - const opts = consumerOpts() - .durable("me") - .ackExplicit() - .filterSubject("a.b") - .callback((_err, msg) => { - msg?.ack(); - }); - - const sub = await js.pullSubscribe("a.>", opts); - const ci = await sub.consumerInfo(); - assertEquals(ci.config.filter_subject, "a.b"); - - await cleanup(ns, nc); -}); - Deno.test("jetstream - jsmsg decode", async () => { const { ns, nc } = await setup(jetstreamServerConf()); const name = nuid.next(); @@ -4489,226 +1640,3 @@ Deno.test("jetstream - source transforms", async () => { await cleanup(ns, nc); }); - -Deno.test("jetstream - pull consumer deleted", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const d = deferred(); - const js = nc.jetstream(); - - js.pull(name, name, 5000) - .catch((err) => { - d.resolve(err); - }); - await nc.flush(); - await jsm.consumers.delete(name, name); - - const err = await d; - assertEquals(err?.message, "consumer deleted"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch consumer deleted", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const d = deferred(); - const js = nc.jetstream(); - - const iter = js.fetch(name, name, { expires: 5000 }); - (async () => { - for await (const _m of iter) { - // nothing - } - })().catch((err) => { - d.resolve(err); - }); - await nc.flush(); - await jsm.consumers.delete(name, name); - - const err = await d; - assertEquals(err?.message, "consumer deleted"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pullSub cb consumer deleted", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const d = deferred(); - const js = nc.jetstream(); - - const opts = consumerOpts().bind(name, name).callback((err, _m) => { - if (err) { - d.resolve(err); - } - }); - const sub = await js.pullSubscribe(name, opts); - sub.pull({ expires: 5000 }); - await nc.flush(); - await jsm.consumers.delete(name, name); - - const err = await d; - assertEquals(err?.message, "consumer deleted"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pullSub iter consumer deleted", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const d = deferred(); - const js = nc.jetstream(); - - const opts = consumerOpts().bind(name, name); - - const sub = await js.pullSubscribe(name, opts); - (async () => { - for await (const _m of sub) { - // nothing - } - })().catch((err) => { - d.resolve(err); - }); - sub.pull({ expires: 5000 }); - await nc.flush(); - await jsm.consumers.delete(name, name); - - const err = await d; - assertEquals(err?.message, "consumer deleted"); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - fetch sync", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - - await js.publish(name); - await js.publish(name); - - const iter = js.fetch(name, name, { batch: 2, no_wait: true }); - const sync = syncIterator(iter); - assertExists(await sync.next()); - assertExists(await sync.next()); - assertEquals(await sync.next(), null); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - push sync", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_subject: "here", - }); - - const js = nc.jetstream(); - - await js.publish(name); - await js.publish(name); - - const sub = await js.subscribe(name, consumerOpts().bind(name, name)); - const sync = syncIterator(sub); - assertExists(await sync.next()); - assertExists(await sync.next()); - - await cleanup(ns, nc); -}); - -Deno.test("jetstream - pull sync", async () => { - const { ns, nc } = await setup(jetstreamServerConf()); - const name = nuid.next(); - const jsm = await nc.jetstreamManager(); - await jsm.streams.add({ - name, - subjects: [name], - storage: StorageType.Memory, - }); - await jsm.consumers.add(name, { - durable_name: name, - ack_policy: AckPolicy.Explicit, - }); - - const js = nc.jetstream(); - - await js.publish(name); - await js.publish(name); - - const sub = await js.pullSubscribe(name, consumerOpts().bind(name, name)); - sub.pull({ batch: 2, no_wait: true }); - const sync = syncIterator(sub); - - assertExists(await sync.next()); - assertExists(await sync.next()); - // if don't unsubscribe, the call will hang because - // we are waiting for the sub.pull() to happen - sub.unsubscribe(); - assertEquals(await sync.next(), null); - - await cleanup(ns, nc); -});