From d943456bcaa8317b8d3ae55f9a1117a5bf8478ed Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan Date: Tue, 14 May 2024 10:40:42 +0800 Subject: [PATCH 001/265] Add unit test for MQClientAPIExtTest (#8080) --- .../impl/mqclient/MQClientAPIExtTest.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java new file mode 100644 index 00000000000..752bc98eabd --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.impl.mqclient; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.utils.FutureUtils; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; + +@RunWith(MockitoJUnitRunner.class) +public class MQClientAPIExtTest { + MQClientAPIExt mqClientAPIExt; + @Mock + NettyRemotingClient remotingClientMock; + + @Before + public void before() { + mqClientAPIExt = Mockito.spy(new MQClientAPIExt(new ClientConfig(), new NettyClientConfig(), null, null)); + Mockito.when(mqClientAPIExt.getRemotingClient()).thenReturn(remotingClientMock); + Mockito.when(remotingClientMock.invoke(anyString(), any(), anyLong())).thenReturn(FutureUtils.completeExceptionally(new RemotingTimeoutException("addr"))); + } + + @Test + public void sendMessageAsync() { + String topic = "test"; + Message msg = new Message(topic, "test".getBytes()); + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setProducerGroup("test"); + requestHeader.setDefaultTopic("test"); + requestHeader.setDefaultTopicQueueNums(1); + requestHeader.setQueueId(0); + requestHeader.setSysFlag(0); + requestHeader.setBornTimestamp(0L); + requestHeader.setFlag(0); + requestHeader.setProperties("test"); + requestHeader.setReconsumeTimes(0); + requestHeader.setUnitMode(false); + requestHeader.setBatch(false); + CompletableFuture future = mqClientAPIExt.sendMessageAsync("127.0.0.1:10911", "test", msg, requestHeader, 10); + assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingTimeoutException.class); + } +} \ No newline at end of file From 502e2d798e57fd4f40d16a90c44679af5ee7f986 Mon Sep 17 00:00:00 2001 From: hiyo <77013030+miles-ton@users.noreply.github.com> Date: Tue, 14 May 2024 14:45:29 +0800 Subject: [PATCH 002/265] [ISSUE #8118] Remove redundant mod in client (#8119) --- .../consumer/rebalance/AllocateMessageQueueAveragely.java | 2 +- .../apache/rocketmq/client/impl/factory/MQClientInstance.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java index 75e5d1c218b..6f63a6fc607 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java @@ -42,7 +42,7 @@ public List allocate(String consumerGroup, String currentCID, List int startIndex = (mod > 0 && index < mod) ? index * averageSize : index * averageSize + mod; int range = Math.min(averageSize, mqAll.size() - startIndex); for (int i = 0; i < range; i++) { - result.add(mqAll.get((startIndex + i) % mqAll.size())); + result.add(mqAll.get(startIndex + i)); } return result; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 227f3346d0d..f964869ac24 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -1222,8 +1222,7 @@ public String findBrokerAddrByTopic(final String topic) { if (topicRouteData != null) { List brokers = topicRouteData.getBrokerDatas(); if (!brokers.isEmpty()) { - int index = random.nextInt(brokers.size()); - BrokerData bd = brokers.get(index % brokers.size()); + BrokerData bd = brokers.get(random.nextInt(brokers.size())); return bd.selectBrokerAddr(); } } From 03b16aadce572b3908dc09b99347fcf7e6c6ac7c Mon Sep 17 00:00:00 2001 From: fujian-zfj <2573259572@qq.com> Date: Tue, 14 May 2024 14:47:37 +0800 Subject: [PATCH 003/265] [ISSUE #8061] Fix npe in netty remoting client (#8064) --- .../org/apache/rocketmq/remoting/netty/NettyRemotingClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index ede6005f541..1bc5e57db52 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -631,7 +631,7 @@ private Channel getAndCreateChannel(final String addr) throws InterruptedExcepti if (channelFuture == null) { return null; } - return getAndCreateChannelAsync(addr).awaitUninterruptibly().channel(); + return channelFuture.awaitUninterruptibly().channel(); } private ChannelFuture getAndCreateNameserverChannelAsync() throws InterruptedException { From 2fdafd232a45ffd4a72239782248e2d1a91d6abc Mon Sep 17 00:00:00 2001 From: hiyo <77013030+miles-ton@users.noreply.github.com> Date: Wed, 15 May 2024 10:01:21 +0800 Subject: [PATCH 004/265] [ISSUE #8136] Replace with createProcessQueue and remove createProcessQueue(topic) in client (#8139) --- .../apache/rocketmq/client/impl/consumer/RebalanceImpl.java | 4 +--- .../rocketmq/client/impl/consumer/RebalanceLitePullImpl.java | 3 --- .../rocketmq/client/impl/consumer/RebalancePullImpl.java | 4 ---- .../rocketmq/client/impl/consumer/RebalancePushImpl.java | 5 ----- 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 53addc5f50c..711df3a9f08 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -525,7 +525,7 @@ private boolean updateProcessQueueTableInRebalance(final String topic, final Set } this.removeDirtyOffset(mq); - ProcessQueue pq = createProcessQueue(topic); + ProcessQueue pq = createProcessQueue(); pq.setLocked(true); long nextOffset = this.computePullFromWhere(mq); if (nextOffset >= 0) { @@ -779,8 +779,6 @@ public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final Pop public abstract PopProcessQueue createPopProcessQueue(); - public abstract ProcessQueue createProcessQueue(String topicName); - public void removeProcessQueue(final MessageQueue mq) { ProcessQueue prev = this.processQueueTable.remove(mq); if (prev != null) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java index 335d89b7877..330772f22bb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -177,7 +177,4 @@ public PopProcessQueue createPopProcessQueue() { return null; } - public ProcessQueue createProcessQueue(String topicName) { - return createProcessQueue(); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java index 1b5f9766174..e0b682868ab 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -103,8 +103,4 @@ public PopProcessQueue createPopProcessQueue() { return null; } - public ProcessQueue createProcessQueue(String topicName) { - return createProcessQueue(); - } - } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index f28890d306f..59e087c0e04 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -288,11 +288,6 @@ public ProcessQueue createProcessQueue() { return new ProcessQueue(); } - @Override - public ProcessQueue createProcessQueue(String topicName) { - return createProcessQueue(); - } - @Override public PopProcessQueue createPopProcessQueue() { return new PopProcessQueue(); From 8150e13a29f9d5ea7bf8814960389837650164af Mon Sep 17 00:00:00 2001 From: Willhow <65004897+Willhow-Gao@users.noreply.github.com> Date: Thu, 16 May 2024 09:50:25 +0800 Subject: [PATCH 005/265] [ISSUE #8145] Optimize some code style in client module (#8146) --- .../ConsumeMessageOrderlyService.java | 8 ++++-- .../impl/consumer/RebalancePushImpl.java | 4 --- .../client/impl/factory/MQClientInstance.java | 6 ++-- .../impl/producer/DefaultMQProducerImpl.java | 28 +++++++++---------- .../latency/LatencyFaultToleranceImpl.java | 8 ++++++ .../client/trace/AsyncTraceDispatcher.java | 7 +++-- 6 files changed, 34 insertions(+), 27 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java index 36d686048ce..3ca465da70d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -85,6 +85,7 @@ public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsu this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_" + consumerGroupTag)); } + @Override public void start() { if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @@ -96,10 +97,11 @@ public void run() { log.error("scheduleAtFixedRate lockMQPeriodically exception", e); } } - }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS); + }, 1000, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS); } } + @Override public void shutdown(long awaitTerminateMillis) { this.stopped = true; this.scheduledExecutorService.shutdown(); @@ -201,8 +203,8 @@ public void submitConsumeRequest( final List msgs, final ProcessQueue processQueue, final MessageQueue messageQueue, - final boolean dispathToConsume) { - if (dispathToConsume) { + final boolean dispatchToConsume) { + if (dispatchToConsume) { ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue); this.consumeExecutor.submit(consumeRequest); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index 59e087c0e04..fe2f19b2f9a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -140,10 +140,6 @@ public boolean clientRebalance(String topic) { return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly() || MessageModel.BROADCASTING.equals(messageModel); } - public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final PopProcessQueue pq) { - return true; - } - @Override public ConsumeType consumeType() { return ConsumeType.CONSUME_PASSIVELY; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index f964869ac24..1ff35a00d12 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -125,8 +125,8 @@ public class MQClientInstance { private final ConcurrentMap> brokerAddrTable = new ConcurrentHashMap<>(); private final ConcurrentMap> brokerVersionTable = new ConcurrentHashMap<>(); - private final Set brokerSupportV2HeartbeatSet = new HashSet(); - private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap(); + private final Set brokerSupportV2HeartbeatSet = new HashSet<>(); + private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap<>(); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread")); private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @Override @@ -1161,7 +1161,7 @@ public FindBrokerResult findBrokerAddressInSubscribe( Entry entry = map.entrySet().iterator().next(); brokerAddr = entry.getValue(); slave = entry.getKey() != MixAll.MASTER_ID; - found = true; + found = brokerAddr != null; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 5b7bd2dc9dc..6268bcc0a17 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -570,8 +570,8 @@ public void run() { } class BackpressureSendCallBack implements SendCallback { - public boolean isSemaphoreAsyncSizeAquired = false; - public boolean isSemaphoreAsyncNumAquired = false; + public boolean isSemaphoreAsyncSizeAcquired = false; + public boolean isSemaphoreAsyncNumbAcquired = false; public int msgLen; private final SendCallback sendCallback; @@ -581,10 +581,10 @@ public BackpressureSendCallBack(final SendCallback sendCallback) { @Override public void onSuccess(SendResult sendResult) { - if (isSemaphoreAsyncSizeAquired) { + if (isSemaphoreAsyncSizeAcquired) { semaphoreAsyncSendSize.release(msgLen); } - if (isSemaphoreAsyncNumAquired) { + if (isSemaphoreAsyncNumbAcquired) { semaphoreAsyncSendNum.release(); } sendCallback.onSuccess(sendResult); @@ -592,10 +592,10 @@ public void onSuccess(SendResult sendResult) { @Override public void onException(Throwable e) { - if (isSemaphoreAsyncSizeAquired) { + if (isSemaphoreAsyncSizeAcquired) { semaphoreAsyncSendSize.release(msgLen); } - if (isSemaphoreAsyncNumAquired) { + if (isSemaphoreAsyncNumbAcquired) { semaphoreAsyncSendNum.release(); } sendCallback.onException(e); @@ -607,31 +607,31 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final throws MQClientException, InterruptedException { ExecutorService executor = this.getAsyncSenderExecutor(); boolean isEnableBackpressureForAsyncMode = this.getDefaultMQProducer().isEnableBackpressureForAsyncMode(); - boolean isSemaphoreAsyncNumAquired = false; - boolean isSemaphoreAsyncSizeAquired = false; + boolean isSemaphoreAsyncNumbAcquired = false; + boolean isSemaphoreAsyncSizeAcquired = false; int msgLen = msg.getBody() == null ? 1 : msg.getBody().length; try { if (isEnableBackpressureForAsyncMode) { long costTime = System.currentTimeMillis() - beginStartTime; - isSemaphoreAsyncNumAquired = timeout - costTime > 0 + isSemaphoreAsyncNumbAcquired = timeout - costTime > 0 && semaphoreAsyncSendNum.tryAcquire(timeout - costTime, TimeUnit.MILLISECONDS); - if (!isSemaphoreAsyncNumAquired) { + if (!isSemaphoreAsyncNumbAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncNum timeout")); return; } costTime = System.currentTimeMillis() - beginStartTime; - isSemaphoreAsyncSizeAquired = timeout - costTime > 0 + isSemaphoreAsyncSizeAcquired = timeout - costTime > 0 && semaphoreAsyncSendSize.tryAcquire(msgLen, timeout - costTime, TimeUnit.MILLISECONDS); - if (!isSemaphoreAsyncSizeAquired) { + if (!isSemaphoreAsyncSizeAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncSize timeout")); return; } } - sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired; - sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired; + sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired; + sendCallback.isSemaphoreAsyncNumbAcquired = isSemaphoreAsyncNumbAcquired; sendCallback.msgLen = msgLen; executor.submit(runnable); } catch (RejectedExecutionException e) { diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java index f629fe44a87..db8bbd66ef2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java @@ -55,6 +55,7 @@ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetec this.serviceDetector = serviceDetector; } + @Override public void detectByOneRound() { for (Map.Entry item : this.faultItemTable.entrySet()) { FaultItem brokerItem = item.getValue(); @@ -77,6 +78,7 @@ public void detectByOneRound() { } } + @Override public void startDetector() { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override @@ -92,6 +94,7 @@ public void run() { }, 3, 3, TimeUnit.SECONDS); } + @Override public void shutdown() { this.scheduledExecutorService.shutdown(); } @@ -128,6 +131,7 @@ public boolean isAvailable(final String name) { return true; } + @Override public boolean isReachable(final String name) { final FaultItem faultItem = this.faultItemTable.get(name); if (faultItem != null) { @@ -141,10 +145,12 @@ public void remove(final String name) { this.faultItemTable.remove(name); } + @Override public boolean isStartDetectorEnable() { return startDetectorEnable; } + @Override public void setStartDetectorEnable(boolean startDetectorEnable) { this.startDetectorEnable = startDetectorEnable; } @@ -177,10 +183,12 @@ public String toString() { '}'; } + @Override public void setDetectTimeout(final int detectTimeout) { this.detectTimeout = detectTimeout; } + @Override public void setDetectInterval(final int detectInterval) { this.detectInterval = detectInterval; } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index d44f22616f4..1fe19773a5a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -153,6 +153,7 @@ public void setNamespaceV2(String namespaceV2) { this.namespaceV2 = namespaceV2; } + @Override public void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClientException { if (isStarted.compareAndSet(false, true)) { traceProducer.setNamesrvAddr(nameSrvAddr); @@ -330,7 +331,7 @@ class TraceDataSegment { private int currentMsgKeySize; private final String traceTopicName; private final String regionId; - private final List traceTransferBeanList = new ArrayList(); + private final List traceTransferBeanList = new ArrayList<>(); TraceDataSegment(String traceTopicName, String regionId) { this.traceTopicName = traceTopicName; @@ -345,7 +346,7 @@ public void addTraceTransferBean(TraceTransferBean traceTransferBean) { this.currentMsgKeySize = traceTransferBean.getTransKey().stream() .reduce(currentMsgKeySize, (acc, x) -> acc + x.length(), Integer::sum); if (currentMsgSize >= traceProducer.getMaxMessageSize() - 10 * 1000 || currentMsgKeySize >= MAX_MSG_KEY_SIZE) { - List dataToSend = new ArrayList(traceTransferBeanList); + List dataToSend = new ArrayList<>(traceTransferBeanList); AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend); traceExecutor.submit(asyncDataSendTask); this.clear(); @@ -356,7 +357,7 @@ public void sendAllData() { if (this.traceTransferBeanList.isEmpty()) { return; } - List dataToSend = new ArrayList(traceTransferBeanList); + List dataToSend = new ArrayList<>(traceTransferBeanList); AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend); traceExecutor.submit(asyncDataSendTask); From c00fac3c770c436717f9aad1d5c7bc6591c734b2 Mon Sep 17 00:00:00 2001 From: oopooa <41882826+oopooa@users.noreply.github.com> Date: Thu, 16 May 2024 09:56:09 +0800 Subject: [PATCH 006/265] [ISSUE #8148] Fix variable typo --- .../org/apache/rocketmq/broker/BrokerController.java | 2 +- .../rocketmq/store/stats/BrokerStatsManager.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a9dcc0af1f8..76224db5cb5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -408,7 +408,7 @@ public BrokerController( this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig ); - this.brokerStatsManager.setProduerStateGetter(new BrokerStatsManager.StateGetter() { + this.brokerStatsManager.setProducerStateGetter(new BrokerStatsManager.StateGetter() { @Override public boolean online(String instanceId, String group, String topic) { if (getTopicConfigManager().getTopicConfigTable().containsKey(NamespaceUtil.wrapNamespace(instanceId, topic))) { diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java index 489d7b4fbce..c165d333fd0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java @@ -142,7 +142,7 @@ public class BrokerStatsManager { private MomentStatsItemSet momentStatsItemSetFallTime; private final StatisticsManager accountStatManager = new StatisticsManager(); - private StateGetter produerStateGetter; + private StateGetter producerStateGetter; private StateGetter consumerStateGetter; private BrokerConfig brokerConfig; @@ -270,7 +270,7 @@ public boolean online(StatisticsItem item) { String kind = item.getStatKind(); if (ACCOUNT_SEND.equals(kind) || ACCOUNT_SEND_REJ.equals(kind)) { - return produerStateGetter.online(instanceId, group, topic); + return producerStateGetter.online(instanceId, group, topic); } else if (ACCOUNT_RCV.equals(kind) || ACCOUNT_SEND_BACK.equals(kind) || ACCOUNT_SEND_BACK_TO_DLQ.equals(kind) || ACCOUNT_REV_REJ.equals(kind)) { return consumerStateGetter.online(instanceId, group, topic); } @@ -296,12 +296,12 @@ public MomentStatsItemSet getMomentStatsItemSetFallTime() { return momentStatsItemSetFallTime; } - public StateGetter getProduerStateGetter() { - return produerStateGetter; + public StateGetter getProducerStateGetter() { + return producerStateGetter; } - public void setProduerStateGetter(StateGetter produerStateGetter) { - this.produerStateGetter = produerStateGetter; + public void setProducerStateGetter(StateGetter producerStateGetter) { + this.producerStateGetter = producerStateGetter; } public StateGetter getConsumerStateGetter() { From 9bdc9db9c46873f948fad2bdac5cc14aa1918f57 Mon Sep 17 00:00:00 2001 From: oopooa <41882826+oopooa@users.noreply.github.com> Date: Fri, 17 May 2024 08:35:23 +0800 Subject: [PATCH 007/265] [ISSUE #8155] Fix doc typo --- docs/cn/BrokerContainer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/BrokerContainer.md b/docs/cn/BrokerContainer.md index 236439284be..a4de9889f27 100644 --- a/docs/cn/BrokerContainer.md +++ b/docs/cn/BrokerContainer.md @@ -72,7 +72,7 @@ sh mqbrokercontainer -c broker-container.conf ``` mqbrokercontainer脚本路径为distribution/bin/mqbrokercontainer。 -## 运行时增加或较少Broker +## 运行时增加或减少Broker 当BrokerContainer进程启动后，也可以通过Admin命令来增加或减少Broker。 From 256217fb6283f8b4c535045682aaef6346be2a87 Mon Sep 17 00:00:00 2001 From: superdev42 <138118491+superdev42@users.noreply.github.com> Date: Mon, 20 May 2024 09:57:21 +0800 Subject: [PATCH 008/265] [ISSUE#8142] Show time of create topic and subScriptionGroup (#8143) * show time of create topic and subScriptionGroup * show time of create topic and subScriptionGroup again * show time of create topic and subScriptionGroup changed --------- Co-authored-by: wengxiaolong <22260113@zju.edu.cn> --- .../broker/processor/AdminBrokerProcessor.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 78a5ba92eed..a1a6f5bf6ce 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -455,6 +455,7 @@ public boolean rejectRequest() { private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + long startTime = System.currentTimeMillis(); final RemotingCommand response = RemotingCommand.createResponseCommand(null); final CreateTopicRequestHeader requestHeader = (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); @@ -514,8 +515,10 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext LOGGER.error("Update / create topic failed for [{}]", request, e); response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark(e.getMessage()); + return response; } - + long executionTime = System.currentTimeMillis() - startTime; + LOGGER.info("executionTime of create topic:{} is {} ms" , topic, executionTime); return response; } @@ -1450,6 +1453,7 @@ public void onException(Throwable e) { private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + long startTime = System.currentTimeMillis(); final RemotingCommand response = RemotingCommand.createResponseCommand(null); LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}", @@ -1462,6 +1466,8 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c response.setCode(ResponseCode.SUCCESS); response.setRemark(null); + long executionTime = System.currentTimeMillis() - startTime; + LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms" ,config.getGroupName() ,executionTime); return response; } @@ -3154,7 +3160,7 @@ private boolean validateSlave(RemotingCommand response) { } private boolean validateBlackListConfigExist(Properties properties) { - for (String blackConfig:configBlackList) { + for (String blackConfig : configBlackList) { if (properties.containsKey(blackConfig)) { return true; } From 0ad0244fe504d9e20a15e83222826f1172bdc4b3 Mon Sep 17 00:00:00 2001 From: hiyo <77013030+miles-ton@users.noreply.github.com> Date: Mon, 20 May 2024 10:47:59 +0800 Subject: [PATCH 009/265] [ISSUE #8164] Log more accurate for the MQClientInstance#doRebalance (#8165) --- .../apache/rocketmq/client/impl/factory/MQClientInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 1ff35a00d12..b4ebf692736 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -1070,7 +1070,7 @@ public boolean doRebalance() { balanced = false; } } catch (Throwable e) { - log.error("doRebalance exception", e); + log.error("doRebalance for consumer group [{}] exception", entry.getKey(), e); } } } From 94bb64f73160e309438fb000719fff0619355dd4 Mon Sep 17 00:00:00 2001 From: mxsm Date: Tue, 21 May 2024 08:32:20 +0800 Subject: [PATCH 010/265] [ISSUE #8162]Optimize the logging printout for the ConfigManager#loadBak method (#8163) --- .../main/java/org/apache/rocketmq/common/ConfigManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java index 5e997596194..099f0d8d560 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java +++ b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java @@ -52,8 +52,8 @@ public boolean load() { private boolean loadBak() { String fileName = null; try { - fileName = this.configFilePath(); - String jsonString = MixAll.file2String(fileName + ".bak"); + fileName = this.configFilePath() + ".bak"; + String jsonString = MixAll.file2String(fileName); if (jsonString != null && jsonString.length() > 0) { this.decode(jsonString); log.info("load " + fileName + " OK"); From 1b42515093fb56a2cabfa754564397e343a357be Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Wed, 22 May 2024 14:09:00 +0800 Subject: [PATCH 011/265] [ISSUE #8129] Support topic reserved time in tiered storage (#8130) Co-authored-by: yuzhou --- broker/BUILD.bazel | 2 ++ .../broker/topic/TopicConfigManager.java | 31 +++++++++++++++++++ .../rocketmq/common/TopicAttributes.java | 9 ++++++ tieredstore/README.md | 4 +-- .../tieredstore/file/FlatFileStore.java | 4 +-- .../tieredstore/file/FlatMessageFile.java | 7 +++++ .../metrics/TieredStoreMetricsManager.java | 5 +-- 7 files changed, 56 insertions(+), 6 deletions(-) diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 785b7657740..0dbc85f9453 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -29,6 +29,7 @@ java_library( "//remoting", "//srvutil", "//store", + "//tieredstore", "@maven//:ch_qos_logback_logback_classic", "@maven//:com_alibaba_fastjson", "@maven//:com_alibaba_fastjson2_fastjson2", @@ -81,6 +82,7 @@ java_library( "//filter", "//remoting", "//store", + "//tieredstore", "@maven//:com_alibaba_fastjson", "@maven//:com_alibaba_fastjson2_fastjson2", "@maven//:com_google_guava_guava", diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 511d29e12ad..1ed9cbab5f8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -51,6 +51,9 @@ import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; +import org.apache.rocketmq.tieredstore.TieredMessageStore; +import org.apache.rocketmq.tieredstore.metadata.MetadataStore; +import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata; import static com.google.common.base.Preconditions.checkNotNull; @@ -501,6 +504,7 @@ public void updateTopicConfig(final TopicConfig topicConfig) { ImmutableMap.copyOf(newAttributes)); topicConfig.setAttributes(finalAttributes); + updateTieredStoreTopicMetadata(topicConfig, newAttributes); TopicConfig old = putTopicConfig(topicConfig); if (old != null) { @@ -515,6 +519,33 @@ public void updateTopicConfig(final TopicConfig topicConfig) { this.persist(topicConfig.getTopicName(), topicConfig); } + private synchronized void updateTieredStoreTopicMetadata(final TopicConfig topicConfig, Map newAttributes) { + if (!(brokerController.getMessageStore() instanceof TieredMessageStore)) { + if (newAttributes.get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()) != null) { + throw new IllegalArgumentException("Update topic reserveTime not supported"); + } + return; + } + + String topic = topicConfig.getTopicName(); + long reserveTime = TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getDefaultValue(); + String attr = topicConfig.getAttributes().get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()); + if (attr != null) { + reserveTime = Long.parseLong(attr); + } + + log.info("Update tiered storage metadata, topic {}, reserveTime {}", topic, reserveTime); + TieredMessageStore tieredMessageStore = (TieredMessageStore) brokerController.getMessageStore(); + MetadataStore metadataStore = tieredMessageStore.getMetadataStore(); + TopicMetadata topicMetadata = metadataStore.getTopic(topic); + if (topicMetadata == null) { + metadataStore.addTopic(topic, reserveTime); + } else if (topicMetadata.getReserveTime() != reserveTime) { + topicMetadata.setReserveTime(reserveTime); + metadataStore.updateTopic(topicMetadata); + } + } + public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) { if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) { diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java b/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java index 1f26866e5b3..c507748c677 100644 --- a/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java +++ b/common/src/main/java/org/apache/rocketmq/common/TopicAttributes.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.rocketmq.common.attribute.Attribute; import org.apache.rocketmq.common.attribute.EnumAttribute; +import org.apache.rocketmq.common.attribute.LongRangeAttribute; import org.apache.rocketmq.common.attribute.TopicMessageType; import static com.google.common.collect.Sets.newHashSet; @@ -43,6 +44,13 @@ public class TopicAttributes { TopicMessageType.topicMessageTypeSet(), TopicMessageType.NORMAL.getValue() ); + public static final LongRangeAttribute TOPIC_RESERVE_TIME_ATTRIBUTE = new LongRangeAttribute( + "reserve.time", + true, + -1, + Long.MAX_VALUE, + -1 + ); public static final Map ALL; @@ -51,5 +59,6 @@ public class TopicAttributes { ALL.put(QUEUE_TYPE_ATTRIBUTE.getName(), QUEUE_TYPE_ATTRIBUTE); ALL.put(CLEANUP_POLICY_ATTRIBUTE.getName(), CLEANUP_POLICY_ATTRIBUTE); ALL.put(TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), TOPIC_MESSAGE_TYPE_ATTRIBUTE); + ALL.put(TOPIC_RESERVE_TIME_ATTRIBUTE.getName(), TOPIC_RESERVE_TIME_ATTRIBUTE); } } diff --git a/tieredstore/README.md b/tieredstore/README.md index baeb56acc36..41e7458a2a6 100644 --- a/tieredstore/README.md +++ b/tieredstore/README.md @@ -57,8 +57,8 @@ Tiered storage provides some useful metrics, see [RIP-46](https://github.com/apa ## How to contribute -We need community participation to add more backend service providers for tiered storage. [PosixFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java), the implementation provided by default is just an example. People who want to contribute can follow it to implement their own providers, such as S3FileSegment, OSSFileSegment, and MinIOFileSegment. Here are some guidelines: +We need community participation to add more backend service providers for tiered storage. [PosixFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java), the implementation provided by default is just an example. People who want to contribute can follow it to implement their own providers, such as S3FileSegment, OSSFileSegment, and MinIOFileSegment. Here are some guidelines: -1. Extend [TieredFileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java) and implement the methods of [TieredStoreProvider](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java) interface. +1. Extend [FileSegment](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java) and implement the methods of [FileSegmentProvider](https://github.com/apache/rocketmq/blob/develop/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentProvider.java) interface. 2. Record metrics where appropriate. See `rocketmq_tiered_store_provider_rpc_latency`, `rocketmq_tiered_store_provider_upload_bytes`, and `rocketmq_tiered_store_provider_download_bytes` 3. No need to maintain your own cache and avoid polluting the page cache. It is already having the read-ahead cache. diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java index 0d7044a5447..f782d099def 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java @@ -60,9 +60,9 @@ public boolean load() { this.flatFileConcurrentMap.clear(); this.recover(); this.executor.commonExecutor.scheduleWithFixedDelay(() -> { - long expiredTimeStamp = System.currentTimeMillis() - - TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); for (FlatMessageFile flatFile : deepCopyFlatFileToList()) { + long expiredTimeStamp = System.currentTimeMillis() - + TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours()); flatFile.destroyExpiredFile(expiredTimeStamp); if (flatFile.consumeQueue.fileSegmentTable.isEmpty()) { this.destroyFile(flatFile.getMessageQueue()); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java index a214059442b..d5675976cb1 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatMessageFile.java @@ -399,4 +399,11 @@ public void destroy() { fileLock.unlock(); } } + + public long getFileReservedHours() { + if (topicMetadata.getReserveTime() > 0) { + return topicMetadata.getReserveTime(); + } + return storeConfig.getTieredStoreFileReservedTime(); + } } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java index e76c86d79bf..8b5a9e63c0c 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.message.MessageQueue; @@ -180,7 +181,7 @@ public static void init(Meter meter, Supplier attributesBuild MessageQueue mq = flatFile.getMessageQueue(); long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > (long) storeConfig.getTieredStoreFileReservedTime() * 60 * 60 * 1000) { + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { continue; } @@ -209,7 +210,7 @@ public static void init(Meter meter, Supplier attributesBuild MessageQueue mq = flatFile.getMessageQueue(); long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > (long) storeConfig.getTieredStoreFileReservedTime() * 60 * 60 * 1000) { + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { continue; } From dcc88c65f1f29b392fbb300001b386dbf1901afc Mon Sep 17 00:00:00 2001 From: Humkum <1109939087@qq.com> Date: Thu, 23 May 2024 13:56:30 +0800 Subject: [PATCH 012/265] [ISSUE #8166] optimize: make compression type configurable in producer clinet level --- .../impl/producer/DefaultMQProducerImpl.java | 28 +------------ .../client/producer/DefaultMQProducer.java | 39 +++++++++++++++++++ .../example/benchmark/BatchProducer.java | 4 +- .../rocketmq/example/benchmark/Producer.java | 4 +- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 6268bcc0a17..7ef34025137 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -70,9 +70,6 @@ import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.compression.CompressionType; -import org.apache.rocketmq.common.compression.Compressor; -import org.apache.rocketmq.common.compression.CompressorFactory; import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; @@ -118,11 +115,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { private MQFaultStrategy mqFaultStrategy; private ExecutorService asyncSenderExecutor; - // compression related - private int compressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); - private CompressionType compressType = CompressionType.of(System.getProperty(MixAll.MESSAGE_COMPRESS_TYPE, "ZLIB")); - private final Compressor compressor = CompressorFactory.getCompressor(compressType); - // backpressure related private Semaphore semaphoreAsyncSendNum; private Semaphore semaphoreAsyncSendSize; @@ -900,7 +892,7 @@ private SendResult sendKernelImpl(final Message msg, boolean msgBodyCompressed = false; if (this.tryToCompressMessage(msg)) { sysFlag |= MessageSysFlag.COMPRESSED_FLAG; - sysFlag |= compressType.getCompressionFlag(); + sysFlag |= this.defaultMQProducer.getCompressType().getCompressionFlag(); msgBodyCompressed = true; } @@ -1070,7 +1062,7 @@ private boolean tryToCompressMessage(final Message msg) { if (body != null) { if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { try { - byte[] data = compressor.compress(body, compressLevel); + byte[] data = this.defaultMQProducer.getCompressor().compress(body, this.defaultMQProducer.getCompressLevel()); if (data != null) { msg.setBody(data); return true; @@ -1763,22 +1755,6 @@ public ConcurrentMap getTopicPublishInfoTable() { return topicPublishInfoTable; } - public int getCompressLevel() { - return compressLevel; - } - - public void setCompressLevel(int compressLevel) { - this.compressLevel = compressLevel; - } - - public CompressionType getCompressType() { - return compressType; - } - - public void setCompressType(CompressionType compressType) { - this.compressType = compressType; - } - public ServiceState getServiceState() { return serviceState; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index b350ba074db..5304887e380 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -36,6 +36,9 @@ import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl; import org.apache.rocketmq.client.trace.hook.SendMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.compression.CompressionType; +import org.apache.rocketmq.common.compression.Compressor; +import org.apache.rocketmq.common.compression.CompressorFactory; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageBatch; import org.apache.rocketmq.common.message.MessageClientIDSetter; @@ -170,6 +173,21 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { private RPCHook rpcHook = null; + /** + * Compress level of compress algorithm. + */ + private int compressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); + + /** + * Compress type of compress algorithm, default using ZLIB. + */ + private CompressionType compressType = CompressionType.of(System.getProperty(MixAll.MESSAGE_COMPRESS_TYPE, "ZLIB")); + + /** + * Compressor of compress algorithm. + */ + private Compressor compressor = CompressorFactory.getCompressor(compressType); + /** * Default constructor. */ @@ -1344,4 +1362,25 @@ public void setStartDetectorEnable(boolean startDetectorEnable) { super.setStartDetectorEnable(startDetectorEnable); this.defaultMQProducerImpl.getMqFaultStrategy().setStartDetectorEnable(startDetectorEnable); } + + public int getCompressLevel() { + return compressLevel; + } + + public void setCompressLevel(int compressLevel) { + this.compressLevel = compressLevel; + } + + public CompressionType getCompressType() { + return compressType; + } + + public void setCompressType(CompressionType compressType) { + this.compressType = compressType; + this.compressor = CompressorFactory.getCompressor(compressType); + } + + public Compressor getCompressor() { + return compressor; + } } diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java index c4a6162a5f5..21a4b3b7e77 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/BatchProducer.java @@ -102,8 +102,8 @@ public static void main(String[] args) throws MQClientException { String compressType = commandLine.hasOption("ct") ? commandLine.getOptionValue("ct").trim() : "ZLIB"; int compressLevel = commandLine.hasOption("cl") ? Integer.parseInt(commandLine.getOptionValue("cl")) : 5; int compressOverHowMuch = commandLine.hasOption("ch") ? Integer.parseInt(commandLine.getOptionValue("ch")) : 4096; - producer.getDefaultMQProducerImpl().setCompressType(CompressionType.of(compressType)); - producer.getDefaultMQProducerImpl().setCompressLevel(compressLevel); + producer.setCompressType(CompressionType.of(compressType)); + producer.setCompressLevel(compressLevel); producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch); System.out.printf("compressType: %s compressLevel: %s%n", compressType, compressLevel); } else { diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java index 480d16b7581..a945283f576 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java @@ -160,8 +160,8 @@ public void run() { String compressType = commandLine.hasOption("ct") ? commandLine.getOptionValue("ct").trim() : "ZLIB"; int compressLevel = commandLine.hasOption("cl") ? Integer.parseInt(commandLine.getOptionValue("cl")) : 5; int compressOverHowMuch = commandLine.hasOption("ch") ? Integer.parseInt(commandLine.getOptionValue("ch")) : 4096; - producer.getDefaultMQProducerImpl().setCompressType(CompressionType.of(compressType)); - producer.getDefaultMQProducerImpl().setCompressLevel(compressLevel); + producer.setCompressType(CompressionType.of(compressType)); + producer.setCompressLevel(compressLevel); producer.setCompressMsgBodyOverHowmuch(compressOverHowMuch); System.out.printf("compressType: %s compressLevel: %s%n", compressType, compressLevel); } else { From b58eefc1a6c272726f17c16b4bb0c2a2666578fa Mon Sep 17 00:00:00 2001 From: Stephanie0002 <55239858+Stephanie0002@users.noreply.github.com> Date: Thu, 23 May 2024 20:15:20 +0800 Subject: [PATCH 013/265] [ISSUE #8182] Modify variable names to enhance readability #8182 --- .../org/apache/rocketmq/example/operation/Consumer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java index 90f2e133a1c..378a9769975 100644 --- a/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/operation/Consumer.java @@ -36,15 +36,15 @@ public class Consumer { public static void main(String[] args) throws MQClientException { CommandLine commandLine = buildCommandline(args); if (commandLine != null) { - String group = commandLine.getOptionValue('g'); + String subGroup = commandLine.getOptionValue('g'); String topic = commandLine.getOptionValue('t'); - String subscription = commandLine.getOptionValue('s'); + String subExpression = commandLine.getOptionValue('s'); final String returnFailedHalf = commandLine.getOptionValue('f'); - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(subGroup); consumer.setInstanceName(Long.toString(System.currentTimeMillis())); - consumer.subscribe(topic, subscription); + consumer.subscribe(topic, subExpression); consumer.registerMessageListener(new MessageListenerConcurrently() { AtomicLong consumeTimes = new AtomicLong(0); From bdc7c0ab6cb296c736c2f322b840c6da9613d10e Mon Sep 17 00:00:00 2001 From: weihubeats Date: Fri, 24 May 2024 10:24:31 +0800 Subject: [PATCH 014/265] [ISSUE #6873] If dns resolve controller address exception will update controllerAddresses to null (#8180) * Adding null does not update * rolling back * dns resolution failure not updating controllerAddresses --- .../broker/controller/ReplicasManager.java | 7 ++-- .../controller/ReplicasManagerTest.java | 36 ++++++++++++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java index a1d711cb275..c294f860ba3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java @@ -30,7 +30,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; - +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.out.BrokerOuterAPI; @@ -803,7 +803,10 @@ private void scanAvailableControllerAddresses() { private void updateControllerAddr() { if (brokerConfig.isFetchControllerAddrByDnsLookup()) { - this.controllerAddresses = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr()); + List adders = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr()); + if (CollectionUtils.isNotEmpty(adders)) { + this.controllerAddresses = adders; + } } else { final String controllerPaths = this.brokerConfig.getControllerAddr(); final String[] controllers = controllerPaths.split(";"); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java index c863f7ac96c..9f17f2bd593 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerTest.java @@ -17,12 +17,15 @@ package org.apache.rocketmq.broker.controller; +import com.google.common.collect.Lists; import java.io.File; +import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; - import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.slave.SlaveSynchronize; @@ -31,11 +34,11 @@ import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdResponseHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerResponseHeader; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.RunningFlags; @@ -52,6 +55,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -189,11 +193,11 @@ public void changeBrokerRoleTest() { syncStateSetA.add(BROKER_ID_2); // not equal to localAddress Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_2, NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetB)) - .doesNotThrowAnyException(); + .doesNotThrowAnyException(); // equal to localAddress Assertions.assertThatCode(() -> replicasManager.changeBrokerRole(BROKER_ID_1, OLD_MASTER_ADDRESS, NEW_MASTER_EPOCH, OLD_MASTER_EPOCH, syncStateSetA)) - .doesNotThrowAnyException(); + .doesNotThrowAnyException(); } @Test @@ -206,6 +210,28 @@ public void changeToMasterTest() { @Test public void changeToSlaveTest() { Assertions.assertThatCode(() -> replicasManager.changeToSlave(NEW_MASTER_ADDRESS, NEW_MASTER_EPOCH, BROKER_ID_2)) - .doesNotThrowAnyException(); + .doesNotThrowAnyException(); } + + @Test + public void testUpdateControllerAddr() throws Exception { + final String controllerAddr = "192.168.1.1"; + brokerConfig.setFetchControllerAddrByDnsLookup(true); + when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(Lists.newArrayList(controllerAddr)); + Method method = ReplicasManager.class.getDeclaredMethod("updateControllerAddr"); + method.setAccessible(true); + method.invoke(replicasManager); + + List addresses = replicasManager.getControllerAddresses(); + Assertions.assertThat(addresses).contains(controllerAddr); + + // Simulating dns resolution exceptions + when(brokerOuterAPI.dnsLookupAddressByDomain(anyString())).thenReturn(new ArrayList<>()); + + method.invoke(replicasManager); + addresses = replicasManager.getControllerAddresses(); + Assertions.assertThat(addresses).contains(controllerAddr); + + } + } From 9c8fdb715f774440009b85da2edbd2ab0278831d Mon Sep 17 00:00:00 2001 From: Humkum <1109939087@qq.com> Date: Fri, 24 May 2024 17:01:41 +0800 Subject: [PATCH 015/265] [ISSUE #8168] fix: There's no need to retry when async produce already timeout (#8169) --- .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 9b15279cb62..816ae877aca 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -704,9 +704,10 @@ public void operationFail(Throwable throwable) { onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, retryTimesWhenSendFailed, times, ex, context, true, producer); } else { - MQClientException ex = new MQClientException("unknow reseaon", throwable); + MQClientException ex = new MQClientException("unknown reason", throwable); + boolean needRetry = !(throwable instanceof RemotingTooMuchRequestException); onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); + retryTimesWhenSendFailed, times, ex, context, needRetry, producer); } } }); From 152055632d28f813ca166fd42cc8c3c0183a370c Mon Sep 17 00:00:00 2001 From: yx9o Date: Thu, 30 May 2024 10:10:03 +0800 Subject: [PATCH 016/265] [ISSUE #8222] Fix spelling errors in comments (#8224) --- .../org/apache/rocketmq/store/config/MessageStoreConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0ba02e4cb9d..9afc02a0c9c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -50,7 +50,7 @@ public class MessageStoreConfig { // CommitLog file size,default is 1G private int mappedFileSizeCommitLog = 1024 * 1024 * 1024; - // CompactinLog file size, default is 100M + // CompactionLog file size, default is 100M private int compactionMappedFileSize = 100 * 1024 * 1024; // CompactionLog consumeQueue file size, default is 10M From db163b431ea9366d1121d04ded4b660fd8a31624 Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Thu, 30 May 2024 13:52:09 +0800 Subject: [PATCH 017/265] Revert "[ISSUE #7757] Use `CompositeByteBuf` to prevent memory copy. (#7694)" (#8209) This reverts commit 7a36d4d736ae8d6d92658e3bdb18f1cd5c0afdb0. --- .../remoting/netty/FileRegionEncoder.java | 20 ++++--------------- .../remoting/netty/FileRegionEncoderTest.java | 5 ++--- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java index 3522c7965c1..7373a560703 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/FileRegionEncoder.java @@ -18,9 +18,6 @@ package org.apache.rocketmq.remoting.netty; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.CompositeByteBuf; -import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.FileRegion; import io.netty.handler.codec.MessageToByteEncoder; @@ -54,12 +51,9 @@ protected void encode(ChannelHandlerContext ctx, FileRegion msg, final ByteBuf o WritableByteChannel writableByteChannel = new WritableByteChannel() { @Override public int write(ByteBuffer src) { - // To prevent mem_copy. - CompositeByteBuf b = (CompositeByteBuf) out; - // Have to increase writerIndex manually. - ByteBuf unpooled = Unpooled.wrappedBuffer(src); - b.addComponent(true, unpooled); - return unpooled.readableBytes(); + int prev = out.writerIndex(); + out.writeBytes(src); + return out.writerIndex() - prev; } @Override @@ -82,10 +76,4 @@ public void close() throws IOException { msg.transferTo(writableByteChannel, transferred); } } - - @Override - protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, FileRegion msg, boolean preferDirect) throws Exception { - ByteBufAllocator allocator = ctx.alloc(); - return preferDirect ? allocator.compositeDirectBuffer() : allocator.compositeHeapBuffer(); - } -} \ No newline at end of file +} diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java index 0cbe627d801..6c7327f258e 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java @@ -21,15 +21,14 @@ import io.netty.channel.DefaultFileRegion; import io.netty.channel.FileRegion; import io.netty.channel.embedded.EmbeddedChannel; -import org.junit.Assert; -import org.junit.Test; - import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.Random; import java.util.UUID; +import org.junit.Assert; +import org.junit.Test; public class FileRegionEncoderTest { From 7a5ea90cb79cc28ae86c368f22d1ab7d2f6f24cc Mon Sep 17 00:00:00 2001 From: wuyue Date: Fri, 31 May 2024 11:10:05 +0800 Subject: [PATCH 018/265] [ISSUE #8053] Return SYSTEM_BUSY if PutMessageStatus is OS_PAGE_CACHE_BUSY (#8054) --- .../apache/rocketmq/broker/processor/SendMessageProcessor.java | 2 +- .../rocketmq/broker/processor/SendMessageProcessorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 912d502eab2..db5b22888dc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -430,7 +430,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult "the broker's disk is full [" + diskUtil() + "], messages are put to the slave, message store has been shut down, etc."); break; case OS_PAGE_CACHE_BUSY: - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.SYSTEM_BUSY); response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); break; case LMQ_CONSUME_QUEUE_NUM_EXCEEDED: diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index e046c888438..442794dcd26 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -174,7 +174,7 @@ public void testProcessRequest_FlushSlaveTimeout() throws Exception { public void testProcessRequest_PageCacheBusy() throws Exception { when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))). thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)))); - assertPutResult(ResponseCode.SYSTEM_ERROR); + assertPutResult(ResponseCode.SYSTEM_BUSY); } @Test From 40271394e344df631d288b169befe1a4d7001255 Mon Sep 17 00:00:00 2001 From: Stephanie0002 <55239858+Stephanie0002@users.noreply.github.com> Date: Fri, 31 May 2024 17:06:00 +0800 Subject: [PATCH 019/265] [ISSUE #8211] Add two metrics rocketmq_topic_create_execution_time and rocketmq_consumer_group_create_execution_time (#8212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add tow metric createTopicTime and createSubscriptionTime in broker * roll back BrokerConfig.java * Add metric view of createTopicTime and createSubscriptionTime in broker * Add two metric rocketmq_active_topic_number and rocketmq_active_subscription_number Signed-off-by: 黄梓淇 --- .../broker/metrics/BrokerMetricsConstant.java | 3 + .../broker/metrics/BrokerMetricsManager.java | 43 +++++++++ .../broker/metrics/InvocationStatus.java | 33 +++++++ .../processor/AdminBrokerProcessor.java | 90 +++++++++++-------- 4 files changed, 134 insertions(+), 35 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java index 5733aa40bac..0af2ac616c4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java @@ -27,6 +27,8 @@ public class BrokerMetricsConstant { public static final String COUNTER_THROUGHPUT_IN_TOTAL = "rocketmq_throughput_in_total"; public static final String COUNTER_THROUGHPUT_OUT_TOTAL = "rocketmq_throughput_out_total"; public static final String HISTOGRAM_MESSAGE_SIZE = "rocketmq_message_size"; + public static final String HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME = "rocketmq_topic_create_execution_time"; + public static final String HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME = "rocketmq_consumer_group_create_execution_time"; public static final String GAUGE_PRODUCER_CONNECTIONS = "rocketmq_producer_connections"; public static final String GAUGE_CONSUMER_CONNECTIONS = "rocketmq_consumer_connections"; @@ -52,6 +54,7 @@ public class BrokerMetricsConstant { public static final String LABEL_PROCESSOR = "processor"; public static final String LABEL_TOPIC = "topic"; + public static final String LABEL_INVOCATION_STATUS = "invocation_status"; public static final String LABEL_IS_RETRY = "is_retry"; public static final String LABEL_IS_SYSTEM = "is_system"; public static final String LABEL_CONSUMER_GROUP = "consumer_group"; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java index fc7e97bda95..0050a0dcd4c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java @@ -64,6 +64,7 @@ import org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant; import org.slf4j.bridge.SLF4JBridgeHandler; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -92,6 +93,8 @@ import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_PRODUCER_CONNECTIONS; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_FINISH_MSG_LATENCY; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_MESSAGE_SIZE; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_AGGREGATION; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CLUSTER_NAME; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP; @@ -135,6 +138,8 @@ public class BrokerMetricsManager { public static LongCounter throughputInTotal = new NopLongCounter(); public static LongCounter throughputOutTotal = new NopLongCounter(); public static LongHistogram messageSize = new NopLongHistogram(); + public static LongHistogram topicCreateExecuteTime = new NopLongHistogram(); + public static LongHistogram consumerGroupCreateExecuteTime = new NopLongHistogram(); // client connection metrics public static ObservableLongGauge producerConnection = new NopObservableLongGauge(); @@ -381,6 +386,14 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) { 1d * 12 * 60 * 60, //12h 1d * 24 * 60 * 60 //24h ); + + List createTimeBuckets = Arrays.asList( + (double) Duration.ofMillis(10).toMillis(), //10ms + (double) Duration.ofMillis(100).toMillis(), //100ms + (double) Duration.ofSeconds(1).toMillis(), //1s + (double) Duration.ofSeconds(3).toMillis(), //3s + (double) Duration.ofSeconds(5).toMillis() //5s + ); InstrumentSelector messageSizeSelector = InstrumentSelector.builder() .setType(InstrumentType.HISTOGRAM) .setName(HISTOGRAM_MESSAGE_SIZE) @@ -401,6 +414,24 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) { SdkMeterProviderUtil.setCardinalityLimit(commitLatencyViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); providerBuilder.registerView(commitLatencySelector, commitLatencyViewBuilder.build()); + InstrumentSelector createTopicTimeSelector = InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) + .setName(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME) + .build(); + InstrumentSelector createSubGroupTimeSelector = InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) + .setName(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME) + .build(); + ViewBuilder createTopicTimeViewBuilder = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets)); + ViewBuilder createSubGroupTimeViewBuilder = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets)); + // To config the cardinalityLimit for openTelemetry metrics exporting. + SdkMeterProviderUtil.setCardinalityLimit(createTopicTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); + providerBuilder.registerView(createTopicTimeSelector, createTopicTimeViewBuilder.build()); + SdkMeterProviderUtil.setCardinalityLimit(createSubGroupTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); + providerBuilder.registerView(createSubGroupTimeSelector, createSubGroupTimeViewBuilder.build()); + for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { ViewBuilder viewBuilder = selectorViewPair.getObject2(); SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); @@ -482,6 +513,18 @@ private void initRequestMetrics() { .setDescription("Incoming messages size") .ofLongs() .build(); + + topicCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME) + .setDescription("The distribution of create topic time") + .ofLongs() + .setUnit("milliseconds") + .build(); + + consumerGroupCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME) + .setDescription("The distribution of create subscription time") + .ofLongs() + .setUnit("milliseconds") + .build(); } private void initConnectionMetrics() { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java new file mode 100644 index 00000000000..c7501e53d96 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.metrics; + +public enum InvocationStatus { + SUCCESS("success"), + FAILURE("failure"); + + private final String name; + + InvocationStatus(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} \ No newline at end of file diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index a1a6f5bf6ce..40a7a461e89 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -38,6 +38,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import io.opentelemetry.api.common.Attributes; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.AccessValidator; @@ -58,6 +59,8 @@ import org.apache.rocketmq.broker.controller.ReplicasManager; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; +import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; +import org.apache.rocketmq.broker.metrics.InvocationStatus; import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; @@ -212,7 +215,8 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.LibC; - +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS; import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; public class AdminBrokerProcessor implements NettyRequestProcessor { @@ -465,45 +469,46 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext String topic = requestHeader.getTopic(); - TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); - if (!result.isValid()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(result.getRemark()); - return response; - } - if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { - if (TopicValidator.isSystemTopic(topic)) { + long executionTime; + try { + TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); + if (!result.isValid()) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The topic[" + topic + "] is conflict with system topic."); + response.setRemark(result.getRemark()); return response; } - } - - TopicConfig topicConfig = new TopicConfig(topic); - topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); - topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); - topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); - topicConfig.setPerm(requestHeader.getPerm()); - topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); - topicConfig.setOrder(requestHeader.getOrder()); - String attributesModification = requestHeader.getAttributes(); - topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); + if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { + if (TopicValidator.isSystemTopic(topic)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The topic[" + topic + "] is conflict with system topic."); + return response; + } + } - if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED - && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("MIXED message type is not supported."); - return response; - } + TopicConfig topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); + topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); + topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); + topicConfig.setPerm(requestHeader.getPerm()); + topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); + topicConfig.setOrder(requestHeader.getOrder()); + String attributesModification = requestHeader.getAttributes(); + topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); + + if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED + && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("MIXED message type is not supported."); + return response; + } - if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) { - LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}", - requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - response.setCode(ResponseCode.SUCCESS); - return response; - } + if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) { + LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}", + requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + response.setCode(ResponseCode.SUCCESS); + return response; + } - try { this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { this.brokerController.registerSingleTopicAll(topicConfig); @@ -517,7 +522,16 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext response.setRemark(e.getMessage()); return response; } - long executionTime = System.currentTimeMillis() - startTime; + finally { + executionTime = System.currentTimeMillis() - startTime; + InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + Attributes attributes = BrokerMetricsManager.newAttributesBuilder() + .put(LABEL_INVOCATION_STATUS, status.getName()) + .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic)) + .build(); + BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes); + } LOGGER.info("executionTime of create topic:{} is {} ms" , topic, executionTime); return response; } @@ -1468,6 +1482,12 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c response.setRemark(null); long executionTime = System.currentTimeMillis() - startTime; LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms" ,config.getGroupName() ,executionTime); + InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + Attributes attributes = BrokerMetricsManager.newAttributesBuilder() + .put(LABEL_INVOCATION_STATUS, status.getName()) + .build(); + BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes); return response; } From a430796e4b975e9ef724ff61e84e47517e8aee62 Mon Sep 17 00:00:00 2001 From: Stephanie0002 <55239858+Stephanie0002@users.noreply.github.com> Date: Fri, 31 May 2024 17:18:30 +0800 Subject: [PATCH 020/265] [ISSUE #8223] Add two metrics rocketmq_topic_number and rocketmq_consumer_group_number (#8225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add tow metric createTopicTime and createSubscriptionTime in broker * roll back BrokerConfig.java * Add metric view of createTopicTime and createSubscriptionTime in broker * Add two metric rocketmq_active_topic_number and rocketmq_active_subscription_number * Add two metric rocketmq_active_topic_number and rocketmq_active_subscription_number Signed-off-by: 黄梓淇 --------- Signed-off-by: 黄梓淇 Co-authored-by: 黄梓淇 --- .../broker/metrics/BrokerMetricsConstant.java | 2 ++ .../broker/metrics/BrokerMetricsManager.java | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java index 0af2ac616c4..4b319f12f6f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java @@ -21,6 +21,8 @@ public class BrokerMetricsConstant { public static final String GAUGE_PROCESSOR_WATERMARK = "rocketmq_processor_watermark"; public static final String GAUGE_BROKER_PERMISSION = "rocketmq_broker_permission"; + public static final String GAUGE_TOPIC_NUM = "rocketmq_topic_number"; + public static final String GAUGE_CONSUMER_GROUP_NUM = "rocketmq_consumer_group_number"; public static final String COUNTER_MESSAGES_IN_TOTAL = "rocketmq_messages_in_total"; public static final String COUNTER_MESSAGES_OUT_TOTAL = "rocketmq_messages_out_total"; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java index 0050a0dcd4c..d8d94f8e69a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java @@ -81,6 +81,8 @@ import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_ROLLBACK_MESSAGES_TOTAL; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_IN_TOTAL; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_OUT_TOTAL; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_TOPIC_NUM; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_GROUP_NUM; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_BROKER_PERMISSION; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_CONNECTIONS; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_INFLIGHT_MESSAGES; @@ -131,6 +133,9 @@ public class BrokerMetricsManager { // broker stats metrics public static ObservableLongGauge processorWatermark = new NopObservableLongGauge(); public static ObservableLongGauge brokerPermission = new NopObservableLongGauge(); + public static ObservableLongGauge topicNum = new NopObservableLongGauge(); + public static ObservableLongGauge consumerGroupNum = new NopObservableLongGauge(); + // request metrics public static LongCounter messagesInTotal = new NopLongCounter(); @@ -490,6 +495,16 @@ private void initStatsMetrics() { .setDescription("Broker permission") .ofLongs() .buildWithCallback(measurement -> measurement.record(brokerConfig.getBrokerPermission(), newAttributesBuilder().build())); + + topicNum = brokerMeter.gaugeBuilder(GAUGE_TOPIC_NUM) + .setDescription("Active topic number") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(brokerController.getTopicConfigManager().getTopicConfigTable().size(), newAttributesBuilder().build())); + + consumerGroupNum = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_GROUP_NUM) + .setDescription("Active subscription group number") + .ofLongs() + .buildWithCallback(measurement -> measurement.record(brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().size(), newAttributesBuilder().build())); } private void initRequestMetrics() { From 144b22ba7d557619275a7b4a343c7350836b0b2c Mon Sep 17 00:00:00 2001 From: mxsm Date: Sun, 2 Jun 2024 07:13:15 +0800 Subject: [PATCH 021/265] [ISSUE #8235]Add @Override annotation for handleDiskFlush method (#8236) --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index cc29cca5d94..1174eca1bab 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -2116,7 +2116,8 @@ public DefaultFlushManager() { this.commitRealTimeService = new CommitLog.CommitRealTimeService(); } - @Override public void start() { + @Override + public void start() { this.flushCommitLogService.start(); if (defaultMessageStore.isTransientStorePoolEnable()) { @@ -2124,6 +2125,7 @@ public DefaultFlushManager() { } } + @Override public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) { // Synchronization flush From 949991e0aabb0e05b78a087292a1b4e0f3e969cf Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:38:33 +0800 Subject: [PATCH 022/265] [ISSUE #8241] Remove duplicate code --- .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 3ac33156b04..3e832e5a9a3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -581,8 +581,6 @@ public void onSuccess(PopResult popResult) { DefaultMQPushConsumerImpl.this.executePopPullRequestImmediately(popRequest); break; case POLLING_FULL: - DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException); - break; default: DefaultMQPushConsumerImpl.this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException); break; From 7145964b4a8db5d29a0af51b352efa2c342122aa Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Tue, 4 Jun 2024 13:46:16 +0800 Subject: [PATCH 023/265] [ISSUE 8230] fix the acl for NotifyClientTerminationRequest because group can be null. (#8231) --- .../org/apache/rocketmq/acl/plain/PlainAccessResource.java | 5 ++++- .../builder/DefaultAuthorizationContextBuilder.java | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java index ccf2418e409..ef05fa6adbb 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java @@ -40,6 +40,7 @@ import java.util.SortedMap; import java.util.TreeMap; import org.apache.commons.codec.DecoderException; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.rocketmq.acl.AccessResource; import org.apache.rocketmq.acl.common.AclException; @@ -268,7 +269,9 @@ public static PlainAccessResource parse(GeneratedMessageV3 messageV3, Authentica } } else if (NotifyClientTerminationRequest.getDescriptor().getFullName().equals(rpcFullName)) { NotifyClientTerminationRequest request = (NotifyClientTerminationRequest) messageV3; - accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB); + if (StringUtils.isNotBlank(request.getGroup().getName())) { + accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB); + } } else if (QueryRouteRequest.getDescriptor().getFullName().equals(rpcFullName)) { QueryRouteRequest request = (QueryRouteRequest) messageV3; accessResource.addResourceAndPerm(request.getTopic(), Permission.ANY); diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java index d6d1556ca20..02d5df236f5 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java @@ -129,7 +129,9 @@ public List build(Metadata metadata, GeneratedMessa } if (message instanceof NotifyClientTerminationRequest) { NotifyClientTerminationRequest request = (NotifyClientTerminationRequest) message; - result = newGroupSubContexts(metadata, request.getGroup()); + if (StringUtils.isNotBlank(request.getGroup().getName())) { + result = newGroupSubContexts(metadata, request.getGroup()); + } } if (message instanceof ChangeInvisibleDurationRequest) { ChangeInvisibleDurationRequest request = (ChangeInvisibleDurationRequest) message; From 7558850df1773ffa67abae4f1b2ceaa7d5060e09 Mon Sep 17 00:00:00 2001 From: liuzc9 <90489940+liuzc9@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:10:56 +0800 Subject: [PATCH 024/265] [ISSUE #8245]Fix typo in user_guide.md --- docs/cn/msg_trace/user_guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/msg_trace/user_guide.md b/docs/cn/msg_trace/user_guide.md index 9cf139fd347..a04c2601f48 100644 --- a/docs/cn/msg_trace/user_guide.md +++ b/docs/cn/msg_trace/user_guide.md @@ -103,7 +103,7 @@ RocketMQ的消息轨迹特性支持两种存储轨迹数据的方式： ### 4.4 使用mqadmin命令发送和查看轨迹 - 发送消息 ```shell -./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your meesgae content" +./mqadmin sendMessage -m true --topic some-topic-name -n 127.0.0.1:9876 -p "your message content" ``` - 查询轨迹 ```shell From 155bcbf23cf026d94a3233d5944571c3532a130d Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 6 Jun 2024 15:50:52 +0800 Subject: [PATCH 025/265] [ISSUE #8197] Support fast filter message in tiered storage (#8198) --- .../core/MessageStoreFetcherImpl.java | 39 ++++--- .../core/MessageStoreFetcherImplTest.java | 102 +++++++++++++++++- 2 files changed, 125 insertions(+), 16 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java index 4ecf79658ee..b72ebe86241 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java @@ -50,7 +50,7 @@ public class MessageStoreFetcherImpl implements MessageStoreFetcher { private static final Logger log = LoggerFactory.getLogger(MessageStoreUtil.TIERED_STORE_LOGGER_NAME); - private static final String CACHE_KEY_FORMAT = "%s@%d@%d"; + protected static final String CACHE_KEY_FORMAT = "%s@%d@%d"; private final String brokerName; private final MetadataStore metadataStore; @@ -113,22 +113,36 @@ protected SelectBufferResult getMessageFromCache(FlatMessageFile flatFile, long buffer.getByteBuffer().asReadOnlyBuffer(), buffer.getStartOffset(), buffer.getSize(), buffer.getTagCode()); } - protected GetMessageResultExt getMessageFromCache(FlatMessageFile flatFile, long offset, int maxCount) { + protected GetMessageResultExt getMessageFromCache( + FlatMessageFile flatFile, long offset, int maxCount, MessageFilter messageFilter) { GetMessageResultExt result = new GetMessageResultExt(); - for (long i = offset; i < offset + maxCount; i++) { - SelectBufferResult buffer = getMessageFromCache(flatFile, i); + int interval = storeConfig.getReadAheadMessageCountThreshold(); + for (long current = offset, end = offset + interval; current < end; current++) { + SelectBufferResult buffer = getMessageFromCache(flatFile, current); if (buffer == null) { + result.setNextBeginOffset(current); break; } + result.setNextBeginOffset(current + 1); + if (messageFilter != null) { + if (!messageFilter.isMatchedByConsumeQueue(buffer.getTagCode(), null)) { + continue; + } + if (!messageFilter.isMatchedByCommitLog(buffer.getByteBuffer().slice(), null)) { + continue; + } + } SelectMappedBufferResult bufferResult = new SelectMappedBufferResult( buffer.getStartOffset(), buffer.getByteBuffer(), buffer.getSize(), null); - result.addMessageExt(bufferResult, i, buffer.getTagCode()); + result.addMessageExt(bufferResult, current, buffer.getTagCode()); + if (result.getMessageCount() == maxCount) { + break; + } } result.setStatus(result.getMessageCount() > 0 ? - GetMessageStatus.FOUND : GetMessageStatus.OFFSET_OVERFLOW_ONE); + GetMessageStatus.FOUND : GetMessageStatus.NO_MATCHED_MESSAGE); result.setMinOffset(flatFile.getConsumeQueueMinOffset()); result.setMaxOffset(flatFile.getConsumeQueueCommitOffset()); - result.setNextBeginOffset(offset + result.getMessageCount()); return result; } @@ -161,11 +175,11 @@ protected CompletableFuture fetchMessageThenPutToCache( }); } - public CompletableFuture getMessageFromCacheAsync( - FlatMessageFile flatFile, String group, long queueOffset, int maxCount) { + public CompletableFuture getMessageFromCacheAsync( + FlatMessageFile flatFile, String group, long queueOffset, int maxCount, MessageFilter messageFilter) { MessageQueue mq = flatFile.getMessageQueue(); - GetMessageResultExt result = getMessageFromCache(flatFile, queueOffset, maxCount); + GetMessageResultExt result = getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter); if (GetMessageStatus.FOUND.equals(result.getStatus())) { log.debug("MessageFetcher cache hit, group={}, topic={}, queueId={}, offset={}, maxCount={}, resultSize={}, lag={}", @@ -179,7 +193,7 @@ public CompletableFuture getMessageFromCacheAsync( group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, result.getMaxOffset() - result.getNextBeginOffset()); return fetchMessageThenPutToCache(flatFile, queueOffset, storeConfig.getReadAheadMessageCountThreshold()) - .thenApply(maxOffset -> getMessageFromCache(flatFile, queueOffset, maxCount)); + .thenApply(maxOffset -> getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter)); } public CompletableFuture getMessageFromTieredStoreAsync( @@ -333,8 +347,7 @@ public CompletableFuture getMessageAsync( boolean cacheBusy = fetcherCache.estimatedSize() > memoryMaxSize * 0.8; if (storeConfig.isReadAheadCacheEnable() && !cacheBusy) { - return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount) - .thenApply(messageResultExt -> messageResultExt.doFilterMessage(messageFilter)); + return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, messageFilter); } else { return getMessageFromTieredStoreAsync(flatFile, queueOffset, maxCount) .thenApply(messageResultExt -> messageResultExt.doFilterMessage(messageFilter)); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java index 7b8b17d5bbc..fdcdec066fe 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImplTest.java @@ -16,24 +16,33 @@ */ package org.apache.rocketmq.tieredstore.core; +import com.google.common.collect.Sets; import java.io.IOException; +import java.nio.ByteBuffer; import java.time.Duration; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.store.DefaultMessageFilter; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.QueryMessageResult; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.TieredMessageStore; -import org.apache.rocketmq.tieredstore.common.GetMessageResultExt; +import org.apache.rocketmq.tieredstore.common.SelectBufferResult; import org.apache.rocketmq.tieredstore.file.FlatMessageFile; +import org.apache.rocketmq.tieredstore.util.MessageFormatUtilTest; import org.apache.rocketmq.tieredstore.util.MessageStoreUtilTest; import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; + +import static org.apache.rocketmq.tieredstore.core.MessageStoreFetcherImpl.CACHE_KEY_FORMAT; public class MessageStoreFetcherImplTest { @@ -164,8 +173,8 @@ public void getMessageFromCacheTest() throws Exception { AtomicLong offset = new AtomicLong(100L); FlatMessageFile flatFile = dispatcherTest.fileStore.getFlatFile(mq); Awaitility.await().atMost(Duration.ofSeconds(10)).until(() -> { - GetMessageResultExt getMessageResult = - fetcher.getMessageFromCacheAsync(flatFile, groupName, offset.get(), batchSize).join(); + GetMessageResult getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, offset.get(), batchSize, null).join(); offset.set(getMessageResult.getNextBeginOffset()); times.incrementAndGet(); return offset.get() == 200L; @@ -173,6 +182,93 @@ public void getMessageFromCacheTest() throws Exception { Assert.assertEquals(100 / times.get(), batchSize); } + @Test + public void getMessageFromCacheTagFilterTest() throws Exception { + dispatcherTest.dispatchFromCommitLogTest(); + mq = dispatcherTest.mq; + messageStore = dispatcherTest.messageStore; + storeConfig = dispatcherTest.storeConfig; + + storeConfig.setReadAheadCacheEnable(true); + fetcher = new MessageStoreFetcherImpl(messageStore); + + FlatMessageFile flatFile = Mockito.mock(FlatMessageFile.class); + Mockito.when(flatFile.getMessageQueue()).thenReturn(mq); + Mockito.when(flatFile.getConsumeQueueMinOffset()).thenReturn(100L); + Mockito.when(flatFile.getConsumeQueueMaxOffset()).thenReturn(200L); + + for (int i = 100; i < 200; i++) { + ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer(); + SelectBufferResult bufferResult = new SelectBufferResult(buffer, i, buffer.remaining(), i % 2); + fetcher.getFetcherCache().put( + String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), i), bufferResult); + } + + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setSubString("1 || 2"); + subscriptionData.setCodeSet(Sets.newHashSet(1, 2)); + MessageFilter filter = new DefaultMessageFilter(subscriptionData); + + GetMessageResult getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 32, filter).join(); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + Assert.assertEquals(32, getMessageResult.getMessageCount()); + Assert.assertEquals(164L, getMessageResult.getNextBeginOffset()); + + getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, 164L, 32, filter).join(); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + Assert.assertEquals(18, getMessageResult.getMessageCount()); + Assert.assertEquals(200L, getMessageResult.getNextBeginOffset()); + + getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, 200L, 32, filter).join(); + Assert.assertEquals(GetMessageStatus.NO_MATCHED_MESSAGE, getMessageResult.getStatus()); + Assert.assertEquals(200L, getMessageResult.getNextBeginOffset()); + + subscriptionData.setCodeSet(Sets.newHashSet(0)); + filter = new DefaultMessageFilter(subscriptionData); + getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 32, filter).join(); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + Assert.assertEquals(32, getMessageResult.getMessageCount()); + Assert.assertEquals(164L - 1L, getMessageResult.getNextBeginOffset()); + } + + @Test + public void getMessageFromCacheTagFilter2Test() throws Exception { + dispatcherTest.dispatchFromCommitLogTest(); + mq = dispatcherTest.mq; + messageStore = dispatcherTest.messageStore; + storeConfig = dispatcherTest.storeConfig; + + storeConfig.setReadAheadCacheEnable(true); + fetcher = new MessageStoreFetcherImpl(messageStore); + + FlatMessageFile flatFile = Mockito.mock(FlatMessageFile.class); + Mockito.when(flatFile.getMessageQueue()).thenReturn(mq); + Mockito.when(flatFile.getConsumeQueueMinOffset()).thenReturn(100L); + Mockito.when(flatFile.getConsumeQueueMaxOffset()).thenReturn(200L); + + for (int i = 100; i < 200; i++) { + ByteBuffer buffer = MessageFormatUtilTest.buildMockedMessageBuffer(); + SelectBufferResult bufferResult = new SelectBufferResult(buffer, i, buffer.remaining(), i - 100L); + fetcher.getFetcherCache().put( + String.format(CACHE_KEY_FORMAT, mq.getTopic(), mq.getQueueId(), i), bufferResult); + } + + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setSubString("1 || 2"); + subscriptionData.setCodeSet(Sets.newHashSet(10, 20)); + MessageFilter filter = new DefaultMessageFilter(subscriptionData); + + GetMessageResult getMessageResult = + fetcher.getMessageFromCacheAsync(flatFile, groupName, 100L, 2, filter).join(); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + Assert.assertEquals(2, getMessageResult.getMessageCount()); + Assert.assertEquals(121L, getMessageResult.getNextBeginOffset()); + } + @Test public void testGetMessageStoreTimeStampAsync() throws Exception { this.getMessageFromTieredStoreTest(); From d1974c55353488095e5122f5ce361c150611f21a Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 6 Jun 2024 20:19:06 +0800 Subject: [PATCH 026/265] [ISSUE #8269] Support pop consumption filter in long polling service (#8271) --- .../NotifyMessageArrivingListener.java | 11 +++-- .../longpolling/PopLongPollingService.java | 44 ++++++++++++++++--- .../broker/longpolling/PopRequest.java | 25 ++++++++--- .../broker/processor/AckMessageProcessor.java | 13 +++--- .../processor/NotificationProcessor.java | 11 ++++- .../broker/processor/PopMessageProcessor.java | 40 ++++++++++++----- 6 files changed, 107 insertions(+), 37 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java index e55ed2778ac..1ddb9f4f8e6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java @@ -36,9 +36,12 @@ public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHol @Override public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { - this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode, - msgStoreTime, filterBitMap, properties); - this.popMessageProcessor.notifyMessageArriving(topic, queueId); - this.notificationProcessor.notifyMessageArriving(topic, queueId); + + this.pullRequestHoldService.notifyMessageArriving( + topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties); + this.popMessageProcessor.notifyMessageArriving( + topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + this.notificationProcessor.notifyMessageArriving( + topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java index a768fe4b9c4..b5179114f37 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java @@ -35,6 +35,9 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.store.ConsumeQueueExt; +import org.apache.rocketmq.store.MessageFilter; import static org.apache.rocketmq.broker.longpolling.PollingResult.NOT_POLLING; import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_FULL; @@ -147,39 +150,61 @@ public void run() { } public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId) { + this.notifyMessageArrivingWithRetryTopic(topic, queueId, null, 0L, null, null); + } + + public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { String notifyTopic; if (KeyBuilder.isPopRetryTopicV2(topic)) { notifyTopic = KeyBuilder.parseNormalTopic(topic); } else { notifyTopic = topic; } - notifyMessageArriving(notifyTopic, queueId); + notifyMessageArriving(notifyTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); } - public void notifyMessageArriving(final String topic, final int queueId) { + public void notifyMessageArriving(final String topic, final int queueId, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { ConcurrentHashMap cids = topicCidMap.get(topic); if (cids == null) { return; } for (Map.Entry cid : cids.entrySet()) { if (queueId >= 0) { - notifyMessageArriving(topic, cid.getKey(), -1); + notifyMessageArriving(topic, -1, cid.getKey(), tagsCode, msgStoreTime, filterBitMap, properties); } - notifyMessageArriving(topic, cid.getKey(), queueId); + notifyMessageArriving(topic, queueId, cid.getKey(), tagsCode, msgStoreTime, filterBitMap, properties); } } - public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) { + public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { ConcurrentSkipListSet remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, queueId)); if (remotingCommands == null || remotingCommands.isEmpty()) { return false; } + PopRequest popRequest = pollRemotingCommands(remotingCommands); if (popRequest == null) { return false; } + + if (popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) { + boolean match = popRequest.getMessageFilter().isMatchedByConsumeQueue(tagsCode, + new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap)); + if (match && properties != null) { + match = popRequest.getMessageFilter().isMatchedByCommitLog(null, properties); + } + if (!match) { + remotingCommands.add(popRequest); + totalPollingNum.incrementAndGet(); + return false; + } + } + if (brokerController.getBrokerConfig().isEnablePopLog()) { - POP_LOGGER.info("lock release , new msg arrive , wakeUp : {}", popRequest); + POP_LOGGER.info("lock release, new msg arrive, wakeUp: {}", popRequest); } return wakeUp(popRequest); } @@ -221,6 +246,11 @@ public boolean wakeUp(final PopRequest request) { */ public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand, final PollingHeader requestHeader) { + return this.polling(ctx, remotingCommand, requestHeader, null, null); + } + + public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand, + final PollingHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter) { if (requestHeader.getPollTime() <= 0 || this.isStopped()) { return NOT_POLLING; } @@ -234,7 +264,7 @@ public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand re } cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE); long expired = requestHeader.getBornTime() + requestHeader.getPollTime(); - final PopRequest request = new PopRequest(remotingCommand, ctx, expired); + final PopRequest request = new PopRequest(remotingCommand, ctx, expired, subscriptionData, messageFilter); boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize(); if (isFull) { POP_LOGGER.info("polling {}, result POLLING_FULL, total:{}", remotingCommand, totalPollingNum.get()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java index a45bcce9f60..0419dbf637d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java @@ -16,28 +16,35 @@ */ package org.apache.rocketmq.broker.longpolling; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.util.Comparator; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; - import org.apache.rocketmq.remoting.protocol.RemotingCommand; - -import io.netty.channel.Channel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.store.MessageFilter; public class PopRequest { private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE); private final RemotingCommand remotingCommand; private final ChannelHandlerContext ctx; - private final long expired; private final AtomicBoolean complete = new AtomicBoolean(false); private final long op = COUNTER.getAndIncrement(); - public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx, long expired) { + private final long expired; + private final SubscriptionData subscriptionData; + private final MessageFilter messageFilter; + + public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx, + long expired, SubscriptionData subscriptionData, MessageFilter messageFilter) { + this.ctx = ctx; this.remotingCommand = remotingCommand; this.expired = expired; + this.subscriptionData = subscriptionData; + this.messageFilter = messageFilter; } public Channel getChannel() { @@ -64,6 +71,14 @@ public long getExpired() { return expired; } + public SubscriptionData getSubscriptionData() { + return subscriptionData; + } + + public MessageFilter getMessageFilter() { + return messageFilter; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("PopRequest{"); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index 9a56498632f..6f7b7e8a24e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -297,15 +297,12 @@ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOf qId, ackOffset, popTime); if (nextOffset > -1) { - if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( - topic, consumeGroup, qId)) { - this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), - consumeGroup, topic, qId, nextOffset); + if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(topic, consumeGroup, qId)) { + this.brokerController.getConsumerOffsetManager().commitOffset( + channel.remoteAddress().toString(), consumeGroup, topic, qId, nextOffset); } - if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, - consumeGroup, qId, invisibleTime)) { - this.brokerController.getPopMessageProcessor().notifyMessageArriving( - topic, consumeGroup, qId); + if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, consumeGroup, qId, invisibleTime)) { + this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup); } } else if (nextOffset == -1) { String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java index 6447500cbe6..c82725fe1e0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java @@ -18,6 +18,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.util.Map; import java.util.Objects; import java.util.Random; import org.apache.rocketmq.broker.BrokerController; @@ -58,8 +59,16 @@ public boolean rejectRequest() { return false; } + // When a new message is written to CommitLog, this method would be called. + // Suspended long polling will receive notification and be wakeup. + public void notifyMessageArriving(final String topic, final int queueId, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + this.popLongPollingService.notifyMessageArrivingWithRetryTopic( + topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + } + public void notifyMessageArriving(final String topic, final int queueId) { - popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId); + this.popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId); } @Override diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 93c04a1b8de..3df4bec9842 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Random; @@ -167,15 +168,23 @@ public ConcurrentLinkedHashMap> getPol } public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) { + this.notifyLongPollingRequestIfNeed( + topic, group, queueId, null, 0L, null, null); + } + + public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId); long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId); long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); long offset = Math.max(popBufferOffset, consumerOffset); if (maxOffset > offset) { - boolean notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, -1); + boolean notifySuccess = popLongPollingService.notifyMessageArriving( + topic, -1, group, tagsCode, msgStoreTime, filterBitMap, properties); if (!notifySuccess) { // notify pop queue - notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, queueId); + notifySuccess = popLongPollingService.notifyMessageArriving( + topic, queueId, group, tagsCode, msgStoreTime, filterBitMap, properties); } this.brokerController.getNotificationProcessor().notifyMessageArriving(topic, queueId); if (this.brokerController.getBrokerConfig().isEnablePopLog()) { @@ -185,12 +194,15 @@ public void notifyLongPollingRequestIfNeed(String topic, String group, int queue } } - public void notifyMessageArriving(final String topic, final int queueId) { - popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId); + public void notifyMessageArriving(final String topic, final int queueId, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + popLongPollingService.notifyMessageArrivingWithRetryTopic( + topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); } - public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) { - return popLongPollingService.notifyMessageArriving(topic, cid, queueId); + public void notifyMessageArriving(final String topic, final int queueId, final String cid) { + popLongPollingService.notifyMessageArriving( + topic, queueId, cid, null, 0L, null, null); } @Override @@ -292,10 +304,11 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC } BrokerConfig brokerConfig = brokerController.getBrokerConfig(); + SubscriptionData subscriptionData = null; ExpressionMessageFilter messageFilter = null; - if (requestHeader.getExp() != null && requestHeader.getExp().length() > 0) { + if (requestHeader.getExp() != null && !requestHeader.getExp().isEmpty()) { try { - SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType()); + subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType()); brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData); @@ -329,7 +342,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC } } else { try { - SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG); + subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG); brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(), requestHeader.getTopic(), subscriptionData); @@ -403,17 +416,20 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC } final RemotingCommand finalResponse = response; + SubscriptionData finalSubscriptionData = subscriptionData; getMessageFuture.thenApply(restNum -> { if (!getMessageResult.getMessageBufferList().isEmpty()) { finalResponse.setCode(ResponseCode.SUCCESS); getMessageResult.setStatus(GetMessageStatus.FOUND); if (restNum > 0) { // all queue pop can not notify specified queue pop, and vice versa - popLongPollingService.notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(), - requestHeader.getQueueId()); + popLongPollingService.notifyMessageArriving( + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(), + null, 0L, null, null); } } else { - PollingResult pollingResult = popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader)); + PollingResult pollingResult = popLongPollingService.polling( + ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter); if (PollingResult.POLLING_SUC == pollingResult) { return null; } else if (PollingResult.POLLING_FULL == pollingResult) { From d9ebe887677646ec8e969c03ea0d394a299894e0 Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan Date: Thu, 6 Jun 2024 20:29:59 +0800 Subject: [PATCH 027/265] [ISSUE #8268] Fix pop orderly commitOffset when NO_MATCHED_MESSAGE (#8270) --- .../broker/processor/PopMessageProcessor.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 3df4bec9842..0304a5dab08 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -638,10 +638,13 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(result.getStatus()) || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(result.getStatus())) && result.getNextBeginOffset() > -1) { - popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset, - requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName()); -// this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic, -// queueId, getMessageTmpResult.getNextBeginOffset()); + if (isOrder) { + this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic, + queueId, result.getNextBeginOffset()); + } else { + popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset, + requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName()); + } } atomicRestNum.set(result.getMaxOffset() - result.getNextBeginOffset() + atomicRestNum.get()); From 4be8fd43720c8635fe135404a7fd000c00bb2a15 Mon Sep 17 00:00:00 2001 From: guyinyou <36399867+guyinyou@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:28:36 +0800 Subject: [PATCH 028/265] [ISSUE #8265] Implement Batch Creation of Topics in RocketMQ Admin (#8267) * add UPDATE_AND_CREATE_TOPIC_LIST * support creating or updating topic config in batch --------- Co-authored-by: guyinyou Co-authored-by: gaoyang.cgy --- .../processor/AdminBrokerProcessor.java | 81 ++++++++++++ .../broker/topic/TopicConfigManager.java | 11 +- .../rocketmq/client/impl/MQClientAPIImpl.java | 20 +++ .../client/impl/MQClientAPIImplTest.java | 23 ++++ .../remoting/protocol/RequestCode.java | 1 + .../body/CreateTopicListRequestBody.java | 42 +++++++ .../header/CreateTopicListRequestHeader.java | 31 +++++ .../tools/admin/DefaultMQAdminExt.java | 5 + .../tools/admin/DefaultMQAdminExtImpl.java | 6 + .../rocketmq/tools/admin/MQAdminExt.java | 3 + .../tools/command/MQAdminStartup.java | 2 + .../topic/UpdateTopicListSubCommand.java | 118 ++++++++++++++++++ .../topic/UpdateTopicListSubCommandTest.java | 41 ++++++ 13 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 40a7a461e89..44bf2a48137 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -116,6 +116,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData; import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -243,6 +244,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, switch (request.getCode()) { case RequestCode.UPDATE_AND_CREATE_TOPIC: return this.updateAndCreateTopic(ctx, request); + case RequestCode.UPDATE_AND_CREATE_TOPIC_LIST: + return this.updateAndCreateTopicList(ctx, request); case RequestCode.DELETE_TOPIC_IN_BROKER: return this.deleteTopic(ctx, request); case RequestCode.GET_ALL_TOPIC_CONFIG: @@ -536,6 +539,84 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext return response; } + private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + long startTime = System.currentTimeMillis(); + + final CreateTopicListRequestBody requestBody = CreateTopicListRequestBody.decode(request.getBody(), CreateTopicListRequestBody.class); + List topicConfigList = requestBody.getTopicConfigList(); + + StringBuilder builder = new StringBuilder(); + for (TopicConfig topicConfig : topicConfigList) { + builder.append(topicConfig.getTopicName()).append(";"); + } + String topicNames = builder.toString(); + LOGGER.info("AdminBrokerProcessor#updateAndCreateTopicList: topicNames: {}, called by {}", topicNames, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + long executionTime; + + try { + // Valid topics + for (TopicConfig topicConfig : topicConfigList) { + String topic = topicConfig.getTopicName(); + TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); + if (!result.isValid()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(result.getRemark()); + return response; + } + if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { + if (TopicValidator.isSystemTopic(topic)) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The topic[" + topic + "] is conflict with system topic."); + return response; + } + } + if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED + && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("MIXED message type is not supported."); + return response; + } + if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) { + LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}", + topic, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + response.setCode(ResponseCode.SUCCESS); + return response; + } + } + + this.brokerController.getTopicConfigManager().updateTopicConfigList(topicConfigList); + if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { + for (TopicConfig topicConfig : topicConfigList) { + this.brokerController.registerSingleTopicAll(topicConfig); + } + } else { + this.brokerController.registerIncrementBrokerData(topicConfigList, this.brokerController.getTopicConfigManager().getDataVersion()); + } + response.setCode(ResponseCode.SUCCESS); + } catch (Exception e) { + LOGGER.error("Update / create topic failed for [{}]", request, e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); + return response; + } + finally { + executionTime = System.currentTimeMillis() - startTime; + InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + Attributes attributes = BrokerMetricsManager.newAttributesBuilder() + .put(LABEL_INVOCATION_STATUS, status.getName()) + .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topicNames)) + .build(); + BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes); + } + LOGGER.info("executionTime of all topics:{} is {} ms" , topicNames, executionTime); + return response; + } + private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 1ed9cbab5f8..d7c06180e9e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -18,6 +18,7 @@ import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -490,7 +491,7 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) } } - public void updateTopicConfig(final TopicConfig topicConfig) { + private void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) { checkNotNull(topicConfig, "topicConfig shouldn't be null"); Map newAttributes = request(topicConfig); @@ -515,10 +516,18 @@ public void updateTopicConfig(final TopicConfig topicConfig) { long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; dataVersion.nextVersion(stateMachineVersion); + } + public void updateTopicConfig(final TopicConfig topicConfig) { + updateSingleTopicConfigWithoutPersist(topicConfig); this.persist(topicConfig.getTopicName(), topicConfig); } + public void updateTopicConfigList(final List topicConfigList) { + topicConfigList.forEach(this::updateSingleTopicConfigWithoutPersist); + this.persist(); + } + private synchronized void updateTieredStoreTopicMetadata(final TopicConfig topicConfig, Map newAttributes) { if (!(brokerController.getMessageStore() instanceof TieredMessageStore)) { if (newAttributes.get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()) != null) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 816ae877aca..f3d7e7c70f9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -118,6 +118,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; import org.apache.rocketmq.remoting.protocol.body.GroupList; @@ -150,6 +151,7 @@ import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.CreateTopicListRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateUserRequestHeader; import org.apache.rocketmq.remoting.protocol.header.DeleteAccessConfigRequestHeader; @@ -430,6 +432,24 @@ public void createTopic(final String addr, final String defaultTopic, final Topi throw new MQClientException(response.getCode(), response.getRemark()); } + public void createTopicList(final String address, final List topicConfigList, final long timeoutMillis) + throws InterruptedException, RemotingException, MQClientException { + CreateTopicListRequestHeader requestHeader = new CreateTopicListRequestHeader(); + CreateTopicListRequestBody requestBody = new CreateTopicListRequestBody(topicConfigList); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, requestHeader); + request.setBody(requestBody.encode()); + + RemotingCommand response = this.remotingClient.invokeSync( + MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis); + assert response != null; + if (response.getCode() == ResponseCode.SUCCESS) { + return; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + public void createPlainAccessConfig(final String addr, final PlainAccessConfig plainAccessConfig, final long timeoutMillis) throws RemotingException, InterruptedException, MQClientException { diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index 97d8d04e648..b0876c7c0d9 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -19,6 +19,7 @@ import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -1068,4 +1069,26 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { int topicCnt = mqClientAPI.addWritePermOfBroker("127.0.0.1", "default-broker", 1000); assertThat(topicCnt).isEqualTo(7); } + + @Test + public void testCreateTopicList_Success() throws RemotingException, InterruptedException, MQClientException { + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; + }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); + + final List topicConfigList = new LinkedList<>(); + for (int i = 0; i < 16; i++) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setTopicName("Topic" + i); + topicConfigList.add(topicConfig); + } + + mqClientAPI.createTopicList(brokerAddr, topicConfigList, 10000); + } + } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index 1de724e0f1e..3be22fc56b7 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -28,6 +28,7 @@ public class RequestCode { public static final int QUERY_CONSUMER_OFFSET = 14; public static final int UPDATE_CONSUMER_OFFSET = 15; public static final int UPDATE_AND_CREATE_TOPIC = 17; + public static final int UPDATE_AND_CREATE_TOPIC_LIST = 18; public static final int GET_ALL_TOPIC_CONFIG = 21; public static final int GET_TOPIC_CONFIG_LIST = 22; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java new file mode 100644 index 00000000000..a72be31ac92 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CreateTopicListRequestBody.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.body; + +import java.util.List; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class CreateTopicListRequestBody extends RemotingSerializable { + @CFNotNull + private List topicConfigList; + + public CreateTopicListRequestBody() {} + + public CreateTopicListRequestBody(List topicConfigList) { + this.topicConfigList = topicConfigList; + } + + public List getTopicConfigList() { + return topicConfigList; + } + + public void setTopicConfigList(List topicConfigList) { + this.topicConfigList = topicConfigList; + } + +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java new file mode 100644 index 00000000000..615de750c48 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CreateTopicListRequestHeader.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.header; + +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.action.RocketMQAction; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.rpc.RpcRequestHeader; + +@RocketMQAction(value = RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, action = Action.CREATE) +public class CreateTopicListRequestHeader extends RpcRequestHeader { + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index a02c878d961..37dd322488f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -195,6 +195,11 @@ public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws R defaultMQAdminExtImpl.createAndUpdateTopicConfig(addr, config); } + @Override + public void createAndUpdateTopicConfigList(String addr, List topicConfigList) throws InterruptedException, RemotingException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateTopicConfigList(addr, topicConfigList); + } + @Override public void createAndUpdatePlainAccessConfig(String addr, PlainAccessConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 2046b1a44cb..b5a20673dab 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -275,6 +275,12 @@ public void createAndUpdateTopicConfig(String addr, this.mqClientInstance.getMQClientAPIImpl().createTopic(addr, this.defaultMQAdminExt.getCreateTopicKey(), config, timeoutMillis); } + @Override + public void createAndUpdateTopicConfigList(final String brokerAddr, + final List topicConfigList) throws RemotingException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createTopicList(brokerAddr, topicConfigList, timeoutMillis); + } + @Override public void createAndUpdatePlainAccessConfig(String addr, PlainAccessConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 50deb7edfc6..96940c38b26 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -92,6 +92,9 @@ void createAndUpdateTopicConfig(final String addr, final TopicConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + void createAndUpdateTopicConfigList(final String addr, + final List topicConfigList) throws InterruptedException, RemotingException, MQClientException; + void createAndUpdatePlainAccessConfig(final String addr, final PlainAccessConfig plainAccessConfig) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index f8b8ec248a8..e785934ba37 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -113,6 +113,7 @@ import org.apache.rocketmq.tools.command.topic.TopicStatusSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateOrderConfCommand; import org.apache.rocketmq.tools.command.topic.UpdateStaticTopicSubCommand; +import org.apache.rocketmq.tools.command.topic.UpdateTopicListSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateTopicPermSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateTopicSubCommand; @@ -187,6 +188,7 @@ public static void main0(String[] args, RPCHook rpcHook) { public static void initCommand() { initCommand(new UpdateTopicSubCommand()); + initCommand(new UpdateTopicListSubCommand()); initCommand(new DeleteTopicSubCommand()); initCommand(new UpdateSubGroupSubCommand()); initCommand(new SetConsumeModeSubCommand()); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java new file mode 100644 index 00000000000..a246059e117 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommand.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.topic; + +import com.alibaba.fastjson2.JSON; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +public class UpdateTopicListSubCommand implements SubCommand { + @Override + public String commandName() { + return "updateTopicList"; + } + + @Override + public String commandDesc() { + return "create or update topic in batch"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + final OptionGroup optionGroup = new OptionGroup(); + Option opt = new Option("b", "brokerAddr", true, "create topic to which broker"); + optionGroup.addOption(opt); + opt = new Option("c", "clusterName", true, "create topic to which cluster"); + optionGroup.addOption(opt); + optionGroup.setRequired(true); + options.addOptionGroup(optionGroup); + + opt = new Option("f", "filename", true, "Path to a file with list of org.apache.rocketmq.common.TopicConfig in json format"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + final DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + final String fileName = commandLine.getOptionValue('f').trim(); + + + try { + final Path filePath = Paths.get(fileName); + if (!Files.exists(filePath)) { + System.out.printf("the file path %s does not exists%n", fileName); + return; + } + final byte[] topicConfigListBytes = Files.readAllBytes(filePath); + final List topicConfigs = JSON.parseArray(topicConfigListBytes, TopicConfig.class); + if (null == topicConfigs || topicConfigs.isEmpty()) { + return; + } + + if (commandLine.hasOption('b')) { + String brokerAddress = commandLine.getOptionValue('b').trim(); + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateTopicConfigList(brokerAddress, topicConfigs); + + System.out.printf("submit batch of topic config to %s success, please check the result later.%n", + brokerAddress); + return; + + } else if (commandLine.hasOption('c')) { + final String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String brokerAddress : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfigList(brokerAddress, topicConfigs); + + System.out.printf("submit batch of topic config to %s success, please check the result later.%n", + brokerAddress); + } + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java new file mode 100644 index 00000000000..95bb579da84 --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.topic; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class UpdateTopicListSubCommandTest { + + @Test + public void testArguments() { + UpdateTopicListSubCommand cmd = new UpdateTopicListSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] {"-b 127.0.0.1:10911", "-f topics.json"}; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new DefaultParser()); + assertEquals("127.0.0.1:10911", commandLine.getOptionValue('b').trim()); + assertEquals("topics.json", commandLine.getOptionValue('f').trim()); + } +} \ No newline at end of file From d60198f621baac54bffb981d7a797a04dfa0bb5a Mon Sep 17 00:00:00 2001 From: rongtong Date: Fri, 7 Jun 2024 13:44:13 +0800 Subject: [PATCH 029/265] [ISSUE #8239] Fix the issue of potential message loss after a crash under synchronous disk flushing configuration. (#8240) --- .../org/apache/rocketmq/store/CommitLog.java | 2 +- .../rocketmq/store/DefaultMessageStore.java | 12 +- .../store/config/MessageStoreConfig.java | 15 +- .../store/ReputMessageServiceTest.java | 148 ++++++++++++++++++ 4 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 1174eca1bab..c2150d7a321 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -651,7 +651,7 @@ public long getConfirmOffset() { } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) { return this.confirmOffset; } else { - return getMaxOffset(); + return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset(); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 97833351d19..a901e850ed6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -2814,7 +2814,11 @@ public long behind() { } public boolean isCommitLogAvailable() { - return this.reputFromOffset < DefaultMessageStore.this.getConfirmOffset(); + return this.reputFromOffset < getReputEndOffset(); + } + + protected long getReputEndOffset() { + return DefaultMessageStore.this.getMessageStoreConfig().isReadUnCommitted() ? DefaultMessageStore.this.commitLog.getMaxOffset() : DefaultMessageStore.this.commitLog.getConfirmOffset(); } public void doReput() { @@ -2834,12 +2838,12 @@ public void doReput() { try { this.reputFromOffset = result.getStartOffset(); - for (int readSize = 0; readSize < result.getSize() && reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) { + for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) { DispatchRequest dispatchRequest = DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false, false); int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize(); - if (reputFromOffset + size > DefaultMessageStore.this.getConfirmOffset()) { + if (reputFromOffset + size > getReputEndOffset()) { doNext = false; break; } @@ -3127,7 +3131,7 @@ public void doReput() { try { this.reputFromOffset = result.getStartOffset(); - for (int readSize = 0; readSize < result.getSize() && reputFromOffset < DefaultMessageStore.this.getConfirmOffset() && doNext; ) { + for (int readSize = 0; readSize < result.getSize() && reputFromOffset < getReputEndOffset() && doNext; ) { ByteBuffer byteBuffer = result.getByteBuffer(); int totalSize = preCheckMessageAndReturnSize(byteBuffer); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 9afc02a0c9c..0060b144cff 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -413,6 +413,12 @@ public class MessageStoreConfig { private int topicQueueLockNum = 32; + /** + * If readUnCommitted is true, the dispatch of the consume queue will exceed the confirmOffset, which may cause the client to read uncommitted messages. + * For example, reput offset exceeding the flush offset during synchronous disk flushing. + */ + private boolean readUnCommitted = false; + public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } @@ -672,7 +678,6 @@ public void setForceVerifyPropCRC(boolean forceVerifyPropCRC) { this.forceVerifyPropCRC = forceVerifyPropCRC; } - public String getStorePathCommitLog() { if (storePathCommitLog == null) { return storePathRootDir + File.separator + "commitlog"; @@ -1819,4 +1824,12 @@ public int getTopicQueueLockNum() { public void setTopicQueueLockNum(int topicQueueLockNum) { this.topicQueueLockNum = topicQueueLockNum; } + + public boolean isReadUnCommitted() { + return readUnCommitted; + } + + public void setReadUnCommitted(boolean readUnCommitted) { + this.readUnCommitted = readUnCommitted; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java new file mode 100644 index 00000000000..d1ce0953331 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/ReputMessageServiceTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store; + +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.UUID; +import java.io.File; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageDecoder; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.config.FlushDiskType; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.any; + +public class ReputMessageServiceTest { + private DefaultMessageStore syncFlushMessageStore; + private DefaultMessageStore asyncFlushMessageStore; + private final String topic = "FooBar"; + private final String tmpdir = System.getProperty("java.io.tmpdir"); + private final String storePathRootParentDir = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + UUID.randomUUID(); + private SocketAddress bornHost; + private SocketAddress storeHost; + + @Before + public void init() throws Exception { + File file = new File(storePathRootParentDir); + UtilAll.deleteFile(file); + syncFlushMessageStore = buildMessageStore(FlushDiskType.SYNC_FLUSH); + asyncFlushMessageStore = buildMessageStore(FlushDiskType.ASYNC_FLUSH); + assertTrue(syncFlushMessageStore.load()); + assertTrue(asyncFlushMessageStore.load()); + syncFlushMessageStore.start(); + asyncFlushMessageStore.start(); + storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + private DefaultMessageStore buildMessageStore(FlushDiskType flushDiskType) throws Exception { + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setHaListenPort(0); + messageStoreConfig.setFlushDiskType(flushDiskType); + messageStoreConfig.setStorePathRootDir(storePathRootParentDir + File.separator + flushDiskType); + messageStoreConfig.setStorePathCommitLog(storePathRootParentDir + File.separator + flushDiskType + File.separator + "commitlog"); + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setLongPollingEnable(false); + DefaultMessageStore messageStore = new DefaultMessageStore(messageStoreConfig, mock(BrokerStatsManager.class), null, brokerConfig, null); + // Mock flush disk service + Field field = CommitLog.class.getDeclaredField("flushManager"); + field.setAccessible(true); + FlushManager flushManager = mock(FlushManager.class); + CompletableFuture completableFuture = new CompletableFuture<>(); + completableFuture.complete(PutMessageStatus.PUT_OK); + when(flushManager.handleDiskFlush(any(AppendMessageResult.class), any(MessageExt.class))).thenReturn(completableFuture); + field.set(messageStore.getCommitLog(), flushManager); + return messageStore; + } + + @Test + public void testReputEndOffset_whenSyncFlush() throws Exception { + for (int i = 0; i < 10; i++) { + assertEquals(PutMessageStatus.PUT_OK, syncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus()); + } + assertEquals(1580, syncFlushMessageStore.getMaxPhyOffset()); + assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere()); + // wait for cq dispatch + Thread.sleep(3000); + assertEquals(0, syncFlushMessageStore.getCommitLog().getFlushedWhere()); + assertEquals(0, syncFlushMessageStore.getMaxOffsetInQueue(topic, 0)); + GetMessageResult getMessageResult = syncFlushMessageStore.getMessage("testGroup", topic, 0, 0, 32, null); + assertEquals(GetMessageStatus.NO_MESSAGE_IN_QUEUE, getMessageResult.getStatus()); + } + + @Test + public void testReputEndOffset_whenAsyncFlush() throws Exception { + for (int i = 0; i < 10; i++) { + assertEquals(PutMessageStatus.PUT_OK, asyncFlushMessageStore.putMessage(buildMessage()).getPutMessageStatus()); + } + assertEquals(1580, asyncFlushMessageStore.getMaxPhyOffset()); + assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere()); + // wait for cq dispatch + Thread.sleep(3000); + assertEquals(0, asyncFlushMessageStore.getCommitLog().getFlushedWhere()); + assertEquals(10, asyncFlushMessageStore.getMaxOffsetInQueue(topic, 0)); + GetMessageResult getMessageResult = asyncFlushMessageStore.getMessage("testGroup", topic, 0, 0, 32, null); + assertEquals(10, getMessageResult.getMessageCount()); + } + + private MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic(topic); + msg.setTags("TAG1"); + msg.setBody("Once, there was a chance for me!".getBytes()); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(0); + msg.setSysFlag(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(storeHost); + msg.setBornHost(bornHost); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + return msg; + } + + @After + public void destroy() throws Exception { + if (this.syncFlushMessageStore != null) { + syncFlushMessageStore.shutdown(); + syncFlushMessageStore.destroy(); + } + if (this.asyncFlushMessageStore != null) { + asyncFlushMessageStore.shutdown(); + asyncFlushMessageStore.destroy(); + } + File file = new File(storePathRootParentDir); + UtilAll.deleteFile(file); + } +} From 3ecc73bcc4f662854549560c12af97480b28bab5 Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:08:32 +0800 Subject: [PATCH 030/265] [ISSUE #8278] Fix fail test (#8279) * fix fail test * fix fail test --- .../tools/command/topic/UpdateTopicListSubCommandTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java index 95bb579da84..71704a7aa8c 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/topic/UpdateTopicListSubCommandTest.java @@ -21,11 +21,11 @@ import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; import org.apache.rocketmq.srvutil.ServerUtil; -import org.junit.jupiter.api.Test; +import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -class UpdateTopicListSubCommandTest { +public class UpdateTopicListSubCommandTest { @Test public void testArguments() { From b96d6b98e836b4604378b8358d58d8a9a1d2037c Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 12 Jun 2024 09:08:49 +0800 Subject: [PATCH 031/265] [ISSUE #8285] Add more test coverage for BrokerPreOnlineService (#8286) --- ...t.java => BrokerPreOnlineServiceTest.java} | 63 ++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) rename container/src/test/java/org/apache/rocketmq/container/{BrokerPreOnlineTest.java => BrokerPreOnlineServiceTest.java} (63%) diff --git a/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java b/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java similarity index 63% rename from container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java rename to container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java index 2158b8d9aca..6a46ed1f6ff 100644 --- a/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineTest.java +++ b/container/src/test/java/org/apache/rocketmq/container/BrokerPreOnlineServiceTest.java @@ -17,15 +17,12 @@ package org.apache.rocketmq.container; -import java.lang.reflect.Field; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPreOnlineService; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; @@ -34,12 +31,21 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class BrokerPreOnlineTest { +public class BrokerPreOnlineServiceTest { @Mock private BrokerContainer brokerContainer; @@ -48,25 +54,27 @@ public class BrokerPreOnlineTest { @Mock private BrokerOuterAPI brokerOuterAPI; - public void init() throws Exception { + public void init(final long brokerId) throws Exception { when(brokerContainer.getBrokerOuterAPI()).thenReturn(brokerOuterAPI); BrokerMemberGroup brokerMemberGroup1 = new BrokerMemberGroup(); Map brokerAddrMap = new HashMap<>(); - brokerAddrMap.put(1L, "127.0.0.1:20911"); + brokerAddrMap.put(0L, "127.0.0.1:10911"); brokerMemberGroup1.setBrokerAddrs(brokerAddrMap); BrokerMemberGroup brokerMemberGroup2 = new BrokerMemberGroup(); brokerMemberGroup2.setBrokerAddrs(new HashMap<>()); - -// when(brokerOuterAPI.syncBrokerMemberGroup(anyString(), anyString())) -// .thenReturn(brokerMemberGroup1) -// .thenReturn(brokerMemberGroup2); -// doNothing().when(brokerOuterAPI).sendBrokerHaInfo(anyString(), anyString(), anyLong(), anyString()); + when(brokerOuterAPI.syncBrokerMemberGroup(anyString(), anyString(), anyBoolean())) + .thenReturn(brokerMemberGroup1) + .thenReturn(brokerMemberGroup2); + BrokerSyncInfo brokerSyncInfo = mock(BrokerSyncInfo.class); + when(brokerOuterAPI.retrieveBrokerHaInfo(anyString())).thenReturn(brokerSyncInfo); DefaultMessageStore defaultMessageStore = mock(DefaultMessageStore.class); when(defaultMessageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); - when(defaultMessageStore.getBrokerConfig()).thenReturn(new BrokerConfig()); + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setBrokerId(brokerId); + when(defaultMessageStore.getBrokerConfig()).thenReturn(brokerConfig); // HAService haService = new DefaultHAService(); // haService.init(defaultMessageStore); @@ -91,11 +99,38 @@ public void init() throws Exception { @Test public void testMasterOnlineConnTimeout() throws Exception { - init(); + init(0); BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController); brokerPreOnlineService.start(); await().atMost(Duration.ofSeconds(30)).until(() -> !innerBrokerController.isIsolated()); } + + @Test + public void testMasterOnlineNormal() throws Exception { + init(0); + BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController); + brokerPreOnlineService.start(); + TimeUnit.SECONDS.sleep(1); + brokerPreOnlineService.shutdown(); + await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped); + } + + @Test + public void testSlaveOnlineNormal() throws Exception { + init(1); + BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController); + brokerPreOnlineService.start(); + TimeUnit.SECONDS.sleep(1); + brokerPreOnlineService.shutdown(); + await().atMost(Duration.ofSeconds(30)).until(brokerPreOnlineService::isStopped); + } + + @Test + public void testGetServiceName() throws Exception { + init(1); + BrokerPreOnlineService brokerPreOnlineService = new BrokerPreOnlineService(innerBrokerController); + assertEquals(BrokerPreOnlineService.class.getSimpleName(), brokerPreOnlineService.getServiceName()); + } } From 1a73e015b8c764a39d2a03a5a58bf26a78638986 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 12 Jun 2024 09:10:33 +0800 Subject: [PATCH 032/265] [ISSUE #8276] Merge duplicate code in DefaultMQProducer constructor (#8277) --- .../client/producer/DefaultMQProducer.java | 42 ++++----- .../producer/DefaultMQProducerTest.java | 92 ++++++++++++++++--- 2 files changed, 99 insertions(+), 35 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 5304887e380..4fd038663b5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -16,13 +16,6 @@ */ package org.apache.rocketmq.client.producer; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.concurrent.ExecutorService; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; @@ -46,11 +39,19 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.ResponseCode; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.ExecutorService; /** * This class is the entry point for applications intending to send messages.

@@ -210,9 +211,7 @@ public DefaultMQProducer(RPCHook rpcHook) { * @param producerGroup Producer group, see the name-sake field. */ public DefaultMQProducer(final String producerGroup) { - this.producerGroup = producerGroup; - defaultMQProducerImpl = new DefaultMQProducerImpl(this, null); - produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); + this(producerGroup, (RPCHook) null); } /** @@ -222,10 +221,7 @@ public DefaultMQProducer(final String producerGroup) { * @param rpcHook RPC hook to execute per each remoting command execution. */ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { - this.producerGroup = producerGroup; - this.rpcHook = rpcHook; - defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); - produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); + this(producerGroup, rpcHook, null); } /** @@ -237,8 +233,7 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { */ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, final List topics) { - this(producerGroup, rpcHook); - this.topics = topics; + this(producerGroup, rpcHook, topics, false, null); } /** @@ -264,9 +259,7 @@ public DefaultMQProducer(final String producerGroup, boolean enableMsgTrace, fin */ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean enableMsgTrace, final String customizedTraceTopic) { - this(producerGroup, rpcHook); - this.enableTrace = enableMsgTrace; - this.traceTopic = customizedTraceTopic; + this(producerGroup, rpcHook, null, enableMsgTrace, customizedTraceTopic); } /** @@ -282,8 +275,13 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, boolean en */ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, final List topics, boolean enableMsgTrace, final String customizedTraceTopic) { - this(producerGroup, rpcHook, enableMsgTrace, customizedTraceTopic); + this.producerGroup = producerGroup; + this.rpcHook = rpcHook; this.topics = topics; + this.enableTrace = enableMsgTrace; + this.traceTopic = customizedTraceTopic; + defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); + produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); } /** diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index d4153c7cd97..7e1fad62477 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -16,19 +16,6 @@ */ package org.apache.rocketmq.client.producer; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -41,9 +28,11 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; @@ -58,13 +47,33 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -596,4 +605,61 @@ public void run() { } return assertionErrors[0]; } + + @Test + public void assertCreateDefaultMQProducer() { + String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp); + assertNotNull(producer1); + assertEquals(producerGroupTemp, producer1.getProducerGroup()); + assertNotNull(producer1.getDefaultMQProducerImpl()); + assertTrue(producer1.getTotalBatchMaxBytes() > 0); + assertTrue(producer1.getBatchMaxBytes() > 0); + assertTrue(producer1.getBatchMaxDelayMs() > 0); + assertNull(producer1.getTopics()); + assertFalse(producer1.isEnableTrace()); + assertTrue(UtilAll.isBlank(producer1.getTraceTopic())); + DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class)); + assertNotNull(producer2); + assertEquals(producerGroupTemp, producer2.getProducerGroup()); + assertNotNull(producer2.getDefaultMQProducerImpl()); + assertTrue(producer2.getTotalBatchMaxBytes() > 0); + assertTrue(producer2.getBatchMaxBytes() > 0); + assertTrue(producer2.getBatchMaxDelayMs() > 0); + assertNull(producer2.getTopics()); + assertFalse(producer2.isEnableTrace()); + assertTrue(UtilAll.isBlank(producer2.getTraceTopic())); + DefaultMQProducer producer3 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList("custom_topic")); + assertNotNull(producer3); + assertEquals(producerGroupTemp, producer3.getProducerGroup()); + assertNotNull(producer3.getDefaultMQProducerImpl()); + assertTrue(producer3.getTotalBatchMaxBytes() > 0); + assertTrue(producer3.getBatchMaxBytes() > 0); + assertTrue(producer3.getBatchMaxDelayMs() > 0); + assertNotNull(producer3.getTopics()); + assertEquals(1, producer3.getTopics().size()); + assertFalse(producer3.isEnableTrace()); + assertTrue(UtilAll.isBlank(producer3.getTraceTopic())); + DefaultMQProducer producer4 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), true, "custom_trace_topic"); + assertNotNull(producer4); + assertEquals(producerGroupTemp, producer4.getProducerGroup()); + assertNotNull(producer4.getDefaultMQProducerImpl()); + assertTrue(producer4.getTotalBatchMaxBytes() > 0); + assertTrue(producer4.getBatchMaxBytes() > 0); + assertTrue(producer4.getBatchMaxDelayMs() > 0); + assertNull(producer4.getTopics()); + assertTrue(producer4.isEnableTrace()); + assertEquals("custom_trace_topic", producer4.getTraceTopic()); + DefaultMQProducer producer5 = new DefaultMQProducer(producerGroupTemp, mock(RPCHook.class), Collections.singletonList("custom_topic"), true, "custom_trace_topic"); + assertNotNull(producer5); + assertEquals(producerGroupTemp, producer5.getProducerGroup()); + assertNotNull(producer5.getDefaultMQProducerImpl()); + assertTrue(producer5.getTotalBatchMaxBytes() > 0); + assertTrue(producer5.getBatchMaxBytes() > 0); + assertTrue(producer5.getBatchMaxDelayMs() > 0); + assertNotNull(producer5.getTopics()); + assertEquals(1, producer5.getTopics().size()); + assertTrue(producer5.isEnableTrace()); + assertEquals("custom_trace_topic", producer5.getTraceTopic()); + } } From 107a8118e5f152f2918aa297c8ad383373b77e7d Mon Sep 17 00:00:00 2001 From: totalo Date: Thu, 13 Jun 2024 10:15:04 +0800 Subject: [PATCH 033/265] Restrict some actions to be triggered only in the official repository (#7695) * build: Restrict the Snapshot Daily Release Automation action to be triggered only in the official repository * build: Restrict the E2E test for pull request action to be triggered only in the official repository * build: Restrict the PUSH-CI action to be triggered only in the official repository * build: update ci --- .github/workflows/pr-e2e-test.yml | 7 +++++-- .github/workflows/push-ci.yml | 5 ++++- .github/workflows/snapshot-automation.yml | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index d0371e31135..9082b6b2227 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -13,10 +13,11 @@ env: jobs: docker: - runs-on: ubuntu-latest if: > + github.repository == 'apache/rocketmq' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: @@ -74,7 +75,9 @@ jobs: path: rocketmq-docker/image-build-ci/versionlist/* list-version: - if: always() + if: > + github.repository == 'apache/rocketmq' && + always() name: List version needs: [docker] runs-on: ubuntu-latest diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index ad29a57c8a8..b679d56d2f0 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -15,6 +15,7 @@ env: jobs: dist-tar: + if: github.repository == 'apache/rocketmq' name: Build dist tar runs-on: ubuntu-latest timeout-minutes: 30 @@ -79,7 +80,9 @@ jobs: list-version: - if: always() + if: > + github.repository == 'apache/rocketmq' && + always() name: List version needs: [docker] runs-on: ubuntu-latest diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml index 88f5f4e0ccb..99855d3aa0d 100644 --- a/.github/workflows/snapshot-automation.yml +++ b/.github/workflows/snapshot-automation.yml @@ -36,6 +36,7 @@ env: jobs: dist-tar: + if: github.repository == 'apache/rocketmq' name: Build dist tar runs-on: ubuntu-latest timeout-minutes: 30 From 6b591556612566c23afeca1ded14c04dfc9dc43d Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:28:02 +0800 Subject: [PATCH 034/265] [ISSUE #7941] add override annotation (#7959) * increase missing annotation * Revert "increase missing annotation" This reverts commit c1f3cef51d781c132d2064e773a58dc496f9c48c. * increase missing annotation --- .../util/data/collect/impl/ListDataCollectorImpl.java | 11 +++++++++++ .../util/data/collect/impl/MapDataCollectorImpl.java | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java index bdd991a335f..b0a1ee3c6ac 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java @@ -39,19 +39,23 @@ public ListDataCollectorImpl(Collection datas) { } } + @Override public Collection getAllData() { return datas; } + @Override public synchronized void resetData() { datas.clear(); unlockIncrement(); } + @Override public long getDataSizeWithoutDuplicate() { return getAllDataWithoutDuplicate().size(); } + @Override public synchronized void addData(Object data) { if (lock) { return; @@ -59,18 +63,22 @@ public synchronized void addData(Object data) { datas.add(data); } + @Override public long getDataSize() { return datas.size(); } + @Override public boolean isRepeatedData(Object data) { return Collections.frequency(datas, data) == 1; } + @Override public synchronized Collection getAllDataWithoutDuplicate() { return new HashSet(datas); } + @Override public int getRepeatedTimeForData(Object data) { int res = 0; for (Object obj : datas) { @@ -81,14 +89,17 @@ public int getRepeatedTimeForData(Object data) { return res; } + @Override public synchronized void removeData(Object data) { datas.remove(data); } + @Override public void lockIncrement() { lock = true; } + @Override public void unlockIncrement() { lock = false; } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java index 899bb855585..7c51af75282 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/MapDataCollectorImpl.java @@ -41,6 +41,7 @@ public MapDataCollectorImpl(Collection datas) { } } + @Override public synchronized void addData(Object data) { if (lock) { return; @@ -52,6 +53,7 @@ public synchronized void addData(Object data) { } } + @Override public Collection getAllData() { List lst = new ArrayList(); for (Entry entry : datas.entrySet()) { @@ -62,15 +64,18 @@ public Collection getAllData() { return lst; } + @Override public long getDataSizeWithoutDuplicate() { return datas.keySet().size(); } + @Override public void resetData() { datas.clear(); unlockIncrement(); } + @Override public long getDataSize() { long sum = 0; for (AtomicInteger count : datas.values()) { @@ -79,6 +84,7 @@ public long getDataSize() { return sum; } + @Override public boolean isRepeatedData(Object data) { if (datas.containsKey(data)) { return datas.get(data).get() == 1; @@ -86,10 +92,12 @@ public boolean isRepeatedData(Object data) { return false; } + @Override public Collection getAllDataWithoutDuplicate() { return datas.keySet(); } + @Override public int getRepeatedTimeForData(Object data) { if (datas.containsKey(data)) { return datas.get(data).intValue(); @@ -97,14 +105,17 @@ public int getRepeatedTimeForData(Object data) { return 0; } + @Override public void removeData(Object data) { datas.remove(data); } + @Override public void lockIncrement() { lock = true; } + @Override public void unlockIncrement() { lock = false; } From 017b7537d2d02fc9a5815eac1f19b8060003fcf4 Mon Sep 17 00:00:00 2001 From: yx9o Date: Thu, 13 Jun 2024 12:09:11 +0800 Subject: [PATCH 035/265] [ISSUE #8227] Optimize DefaultMQPushConsumer construction method (#8228) --- .../consumer/DefaultMQPushConsumer.java | 23 +++----- .../consumer/DefaultMQPushConsumerTest.java | 58 ++++++++++++++----- 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 312f4632cab..38a412c237b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -16,10 +16,6 @@ */ package org.apache.rocketmq.client.consumer; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.consumer.listener.MessageListener; @@ -40,12 +36,17 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.NamespaceUtil; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; /** * In most scenarios, this is the mostly recommended class to consume messages. @@ -328,10 +329,7 @@ public DefaultMQPushConsumer(RPCHook rpcHook) { * @param rpcHook RPC hook to execute before each remoting command. */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook) { - this.consumerGroup = consumerGroup; - this.rpcHook = rpcHook; - this.allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); - defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); + this(consumerGroup, rpcHook, new AllocateMessageQueueAveragely()); } @@ -355,10 +353,7 @@ public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, AllocateMessageQueueStrategy allocateMessageQueueStrategy) { - this.consumerGroup = consumerGroup; - this.rpcHook = rpcHook; - this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; - defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); + this(consumerGroup, rpcHook, allocateMessageQueueStrategy, false, null); } /** diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index 3943b922899..a10fd74b34f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -16,20 +16,6 @@ */ package org.apache.rocketmq.client.consumer; -import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; @@ -37,6 +23,8 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -53,6 +41,7 @@ import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageClientExt; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -62,7 +51,6 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -71,8 +59,27 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -80,6 +87,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -206,7 +214,7 @@ public static void terminate() { @Test public void testStart_OffsetShouldNotNUllAfterStart() { - Assert.assertNotNull(pushConsumer.getOffsetStore()); + assertNotNull(pushConsumer.getOffsetStore()); } @Test @@ -388,4 +396,22 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, pullMessageService.executePullRequestImmediately(createPullRequest()); assertThat(messageExts[0]).isNull(); } + + @Test + public void assertCreatePushConsumer() { + DefaultMQPushConsumer pushConsumer1 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class)); + assertNotNull(pushConsumer1); + assertEquals(consumerGroup, pushConsumer1.getConsumerGroup()); + assertTrue(pushConsumer1.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragely); + assertNotNull(pushConsumer1.defaultMQPushConsumerImpl); + assertFalse(pushConsumer1.isEnableTrace()); + assertTrue(UtilAll.isBlank(pushConsumer1.getTraceTopic())); + DefaultMQPushConsumer pushConsumer2 = new DefaultMQPushConsumer(consumerGroup, mock(RPCHook.class), new AllocateMessageQueueAveragelyByCircle()); + assertNotNull(pushConsumer2); + assertEquals(consumerGroup, pushConsumer2.getConsumerGroup()); + assertTrue(pushConsumer2.getAllocateMessageQueueStrategy() instanceof AllocateMessageQueueAveragelyByCircle); + assertNotNull(pushConsumer2.defaultMQPushConsumerImpl); + assertFalse(pushConsumer2.isEnableTrace()); + assertTrue(UtilAll.isBlank(pushConsumer2.getTraceTopic())); + } } From 3ac5c73cd1fbd715525bff6e6aa9f4bfe768ca30 Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan Date: Thu, 13 Jun 2024 14:48:12 +0800 Subject: [PATCH 036/265] [ISSUE #8281] Optimize pop log level (#8282) * change pop log level to debug when all the revive messages are handled --- .../apache/rocketmq/broker/processor/PopReviveService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 4fab3d500ba..e3ba492f280 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -322,7 +322,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { if (endTime != 0 && System.currentTimeMillis() - endTime > 3 * PopAckConstants.SECOND && timerDelay <= 0 && commitLogDelay <= 0) { endTime = System.currentTimeMillis(); } - POP_LOGGER.info("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ", + POP_LOGGER.debug("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ", queueId, offset, old, endTime, timerDelay, commitLogDelay); if (endTime - firstRt > PopAckConstants.ackTimeInterval + PopAckConstants.SECOND) { break; @@ -528,7 +528,7 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { GetMessageStatus getMessageStatus = resultPair.getObject1(); MessageExt message = resultPair.getObject2(); if (message == null) { - POP_LOGGER.warn("reviveQueueId={}, can not get biz msg topic is {}, offset is {}, then continue", + POP_LOGGER.debug("reviveQueueId={}, can not get biz msg topic is {}, offset is {}, then continue", queueId, popCheckPoint.getTopic(), msgOffset); switch (getMessageStatus) { case MESSAGE_WAS_REMOVING: From 38c56cd84b4079971fd5989bc6702c00c5621472 Mon Sep 17 00:00:00 2001 From: mxsm Date: Thu, 13 Jun 2024 15:31:16 +0800 Subject: [PATCH 037/265] [ISSUE #8293] Remove the redundant code from the MessageDecoder#encodeMessage method (#8294) --- .../java/org/apache/rocketmq/common/message/MessageDecoder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index b053f827597..f5491e192af 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -666,7 +666,6 @@ public static byte[] encodeMessage(Message message) { byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); //note properties length must not more than Short.MAX short propertiesLength = (short) propertiesBytes.length; - int sysFlag = message.getFlag(); int storeSize = 4 // 1 TOTALSIZE + 4 // 2 MAGICCOD + 4 // 3 BODYCRC From 568950bbc846eea24ac6431c15e22f7e506d4905 Mon Sep 17 00:00:00 2001 From: YASH PATEL <121890726+yp969803@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:29:17 +0530 Subject: [PATCH 038/265] [ISSUE #7466] Added fast failure in adminBrokerThreadPoolQueue (#7466) (#7798) --- .../rocketmq/broker/latency/BrokerFastFailure.java | 3 +++ .../java/org/apache/rocketmq/common/BrokerConfig.java | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index 3b6e9dc676e..0135ac929a7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -115,6 +115,9 @@ private void cleanExpiredRequest() { cleanExpiredRequestInQueue(this.brokerController.getAckThreadPoolQueue(), brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue()); + + cleanExpiredRequestInQueue(this.brokerController.getAdminBrokerThreadPoolQueue(), + brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue()); } void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, final long maxWaitTimeMillsInQueue) { diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index d859f965e40..378301bedd2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -148,7 +148,7 @@ public class BrokerConfig extends BrokerIdentity { private long waitTimeMillsInHeartbeatQueue = 31 * 1000; private long waitTimeMillsInTransactionQueue = 3 * 1000; private long waitTimeMillsInAckQueue = 3000; - + private long waitTimeMillsInAdminBrokerQueue = 5 * 1000; private long startAcceptSendRequestTimeStamp = 0L; private boolean traceOn = true; @@ -1167,6 +1167,14 @@ public String getMsgTraceTopicName() { return msgTraceTopicName; } + public long getWaitTimeMillsInAdminBrokerQueue() { + return waitTimeMillsInAdminBrokerQueue; + } + + public void setWaitTimeMillsInAdminBrokerQueue(long waitTimeMillsInAdminBrokerQueue) { + this.waitTimeMillsInAdminBrokerQueue = waitTimeMillsInAdminBrokerQueue; + } + public void setMsgTraceTopicName(String msgTraceTopicName) { this.msgTraceTopicName = msgTraceTopicName; } From 1511809975a3e4f724286fa6e74090ad28598c03 Mon Sep 17 00:00:00 2001 From: yx9o Date: Tue, 18 Jun 2024 09:53:35 +0800 Subject: [PATCH 039/265] [ISSUE #8300] Add more test coverage for DefaultMQProducer (#8301) * [ISSUE #8300] Add more test coverage for DefaultMQProducer * Add more tests. * Update --- .../producer/DefaultMQProducerTest.java | 195 ++++++++++++++++-- 1 file changed, 182 insertions(+), 13 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 7e1fad62477..96086c7a255 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -28,7 +28,9 @@ import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.latency.MQFaultStrategy; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.compression.CompressionType; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -49,6 +51,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -82,15 +85,14 @@ public class DefaultMQProducerTest { private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @Mock private MQClientAPIImpl mQClientAPIImpl; - @Mock - private NettyRemotingClient nettyRemotingClient; private DefaultMQProducer producer; private Message message; private Message zeroMsg; private Message bigMessage; - private String topic = "FooBar"; - private String producerGroupPrefix = "FooBar_PID"; + private final String topic = "FooBar"; + private final String producerGroupPrefix = "FooBar_PID"; + private final long defaultTimeout = 3000L; @Before public void init() throws Exception { @@ -196,7 +198,7 @@ public void onException(Throwable e) { countDownLatch.countDown(); } }); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); } @Test @@ -240,7 +242,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { //this message is send success producer.send(message, sendCallback, 1000); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(5); // off enableBackpressureForAsyncMode @@ -253,7 +255,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { //this message is send success producer.send(message, sendCallback, 1000); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(10); } @@ -301,7 +303,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { // this message is send failed producer.send(msgs, new MessageQueue(), sendCallback, 1000); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(1); // off enableBackpressureForAsyncMode @@ -312,7 +314,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { // this message is send failed producer.send(msgs, new MessageQueue(), sendCallback, 1000); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(2); } @@ -333,7 +335,7 @@ public void onSuccess(SendResult sendResult) { public void onException(Throwable e) { } }); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); } @Test @@ -472,7 +474,7 @@ public void onException(Throwable e) { future.setSendRequestOk(true); future.getRequestCallback().onSuccess(responseMsg); } - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); } @Test @@ -509,7 +511,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { future.getRequestCallback().onException(e); } } - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(1); } @@ -533,7 +535,7 @@ public void onException(Throwable e) { } }); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(defaultTimeout, TimeUnit.MILLISECONDS); producer.setAutoBatch(false); } @@ -662,4 +664,171 @@ public void assertCreateDefaultMQProducer() { assertTrue(producer5.isEnableTrace()); assertEquals("custom_trace_topic", producer5.getTraceTopic()); } + + @Test + public void assertSend() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException { + setDefaultMQProducerImpl(); + setOtherParam(); + SendResult send = producer.send(message, defaultTimeout); + assertNull(send); + Collection msgs = Collections.singletonList(message); + send = producer.send(msgs); + assertNull(send); + send = producer.send(msgs, defaultTimeout); + assertNull(send); + } + + @Test + public void assertSendOneway() throws RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException { + setDefaultMQProducerImpl(); + producer.sendOneway(message); + MessageQueue mq = mock(MessageQueue.class); + producer.sendOneway(message, mq); + MessageQueueSelector selector = mock(MessageQueueSelector.class); + producer.sendOneway(message, selector, 1); + } + + @Test + public void assertSendByQueue() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException { + setDefaultMQProducerImpl(); + MessageQueue mq = mock(MessageQueue.class); + SendResult send = producer.send(message, mq); + assertNull(send); + send = producer.send(message, mq, defaultTimeout); + assertNull(send); + Collection msgs = Collections.singletonList(message); + send = producer.send(msgs, mq); + assertNull(send); + send = producer.send(msgs, mq, defaultTimeout); + assertNull(send); + } + + @Test + public void assertSendByQueueSelector() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException { + setDefaultMQProducerImpl(); + MessageQueueSelector selector = mock(MessageQueueSelector.class); + SendResult send = producer.send(message, selector, 1); + assertNull(send); + send = producer.send(message, selector, 1, defaultTimeout); + assertNull(send); + } + + @Test + public void assertRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException, NoSuchFieldException, IllegalAccessException, RequestTimeoutException { + setDefaultMQProducerImpl(); + MessageQueueSelector selector = mock(MessageQueueSelector.class); + Message replyNsg = producer.request(message, selector, 1, defaultTimeout); + assertNull(replyNsg); + RequestCallback requestCallback = mock(RequestCallback.class); + producer.request(message, selector, 1, requestCallback, defaultTimeout); + MessageQueue mq = mock(MessageQueue.class); + producer.request(message, mq, defaultTimeout); + producer.request(message, mq, requestCallback, defaultTimeout); + } + + @Test(expected = RuntimeException.class) + public void assertSendMessageInTransaction() throws MQClientException { + TransactionSendResult result = producer.sendMessageInTransaction(message, 1); + assertNull(result); + } + + @Test + public void assertSearchOffset() throws MQClientException, NoSuchFieldException, IllegalAccessException { + setDefaultMQProducerImpl(); + MessageQueue mq = mock(MessageQueue.class); + long result = producer.searchOffset(mq, System.currentTimeMillis()); + assertEquals(0L, result); + } + + @Test + public void assertBatchMaxDelayMs() throws NoSuchFieldException, IllegalAccessException { + setProduceAccumulator(true); + assertEquals(0, producer.getBatchMaxDelayMs()); + setProduceAccumulator(false); + assertEquals(10, producer.getBatchMaxDelayMs()); + producer.batchMaxDelayMs(1000); + assertEquals(1000, producer.getBatchMaxDelayMs()); + } + + @Test + public void assertBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException { + setProduceAccumulator(true); + assertEquals(0L, producer.getBatchMaxBytes()); + setProduceAccumulator(false); + assertEquals(32 * 1024L, producer.getBatchMaxBytes()); + producer.batchMaxBytes(64 * 1024L); + assertEquals(64 * 1024L, producer.getBatchMaxBytes()); + } + + @Test + public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAccessException { + setProduceAccumulator(true); + assertEquals(0L, producer.getTotalBatchMaxBytes()); + } + + @Test + public void assertGetRetryResponseCodes() { + assertNotNull(producer.getRetryResponseCodes()); + assertEquals(7, producer.getRetryResponseCodes().size()); + } + + @Test + public void assertIsSendLatencyFaultEnable() { + assertFalse(producer.isSendLatencyFaultEnable()); + } + + @Test + public void assertGetLatencyMax() { + assertNotNull(producer.getLatencyMax()); + } + + @Test + public void assertGetNotAvailableDuration() { + assertNotNull(producer.getNotAvailableDuration()); + } + + @Test + public void assertIsRetryAnotherBrokerWhenNotStoreOK() { + assertFalse(producer.isRetryAnotherBrokerWhenNotStoreOK()); + } + + private void setOtherParam() { + producer.setCreateTopicKey("createTopicKey"); + producer.setRetryAnotherBrokerWhenNotStoreOK(false); + producer.setDefaultTopicQueueNums(6); + producer.setRetryTimesWhenSendFailed(1); + producer.setSendMessageWithVIPChannel(false); + producer.setNotAvailableDuration(new long[1]); + producer.setLatencyMax(new long[1]); + producer.setSendLatencyFaultEnable(false); + producer.setRetryTimesWhenSendAsyncFailed(1); + producer.setTopics(Collections.singletonList(topic)); + producer.setStartDetectorEnable(false); + producer.setCompressLevel(5); + producer.setCompressType(CompressionType.LZ4); + producer.addRetryResponseCode(0); + ExecutorService executorService = mock(ExecutorService.class); + producer.setAsyncSenderExecutor(executorService); + } + + private void setProduceAccumulator(final boolean isDefault) throws NoSuchFieldException, IllegalAccessException { + ProduceAccumulator accumulator = null; + if (!isDefault) { + accumulator = new ProduceAccumulator("instanceName"); + } + setField(producer, "produceAccumulator", accumulator); + } + + private void setDefaultMQProducerImpl() throws NoSuchFieldException, IllegalAccessException { + DefaultMQProducerImpl producerImpl = mock(DefaultMQProducerImpl.class); + setField(producer, "defaultMQProducerImpl", producerImpl); + when(producerImpl.getMqFaultStrategy()).thenReturn(mock(MQFaultStrategy.class)); + } + + private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException { + Class clazz = target.getClass(); + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, newValue); + } } From 4cedd0793f44d46986fb6e43b92c6f4031b64670 Mon Sep 17 00:00:00 2001 From: maclong1989 <814742806@qq.com> Date: Fri, 21 Jun 2024 07:54:15 +0800 Subject: [PATCH 040/265] fix document typo in SlaveActingMasterMode.md (#8315) Signed-off-by: maclong1989 <814742806@qq.com> --- docs/cn/SlaveActingMasterMode.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn/SlaveActingMasterMode.md b/docs/cn/SlaveActingMasterMode.md index b1e266f2b42..b08cf0d9af3 100644 --- a/docs/cn/SlaveActingMasterMode.md +++ b/docs/cn/SlaveActingMasterMode.md @@ -70,9 +70,9 @@ public void changeSpecialServiceStatus(boolean shouldStart) { 2. Broker能及时感知到同组Broker的上下线情况。 -针对1，Nameserver原本就存在判活机制，定时会扫描不活跃的broker使其下线，而原本broker与nameserver的“心跳”则依赖于registerBroker操作，而这个操作涉及到topic信息上报，过于“重”，而且注册间隔过于长，因此需要一个轻量级的心跳机制，RoccketMQ 5.0在nameserver和broker间新增BrokerHeartbeat请求，broker会定时向nameserver发送心跳，若nameserver定时任务扫描发现超过心跳超时时间仍未收到该broker的心跳，将unregister该broker。registerBroker时会完成心跳超时时间的设置，并且注册时如果发现broker组内最小brokerId发生变化，将反向通知该组所有broker，并在路由获取时将最小brokerId的Slave路由替换使其充当只读模式的Master的角色 +针对1，Nameserver原本就存在判活机制，定时会扫描不活跃的broker使其下线，而原本broker与nameserver的“心跳”则依赖于registerBroker操作，而这个操作涉及到topic信息上报，过于“重”，而且注册间隔过于长，因此需要一个轻量级的心跳机制，RocketMQ 5.0在nameserver和broker间新增BrokerHeartbeat请求，broker会定时向nameserver发送心跳，若nameserver定时任务扫描发现超过心跳超时时间仍未收到该broker的心跳，将unregister该broker。registerBroker时会完成心跳超时时间的设置，并且注册时如果发现broker组内最小brokerId发生变化，将反向通知该组所有broker，并在路由获取时将最小brokerId的Slave路由替换使其充当只读模式的Master的角色 -针对2，通过两个机制来及时感知同组broker上下线情况，1是上文中介绍的当nameserver发现该broker组内最小brokerId发生变化，反向通知该组所有broker。2是broker自身会有定时任务，向nameserver同步本broker组存活broker的信息，RoccketMQ 5.0会新增GetBrokerMemberGroup请求来完成该工作。 +针对2，通过两个机制来及时感知同组broker上下线情况，1是上文中介绍的当nameserver发现该broker组内最小brokerId发生变化，反向通知该组所有broker。2是broker自身会有定时任务，向nameserver同步本broker组存活broker的信息，RocketMQ 5.0会新增GetBrokerMemberGroup请求来完成该工作。 Slave Broker发现自己是该组中最小的brokerId，将会开启代理模式，而一旦Master Broker重新上线，Slave Broker同样会通过Nameserver反向通知或自身定时任务同步同组broker的信息感知到，并自动结束代理模式。 From 209184339b1478ea3b3f8c90ecc7146e9cecbf6d Mon Sep 17 00:00:00 2001 From: Mrhorse99 <82942806+Mrhorse99@users.noreply.github.com> Date: Fri, 21 Jun 2024 07:55:01 +0800 Subject: [PATCH 041/265] [ISSUE #8274] Optimize some codestyles and fix some warnings (#8275) --- .../broker/processor/AdminBrokerProcessor.java | 17 ++++++++--------- .../rocketmq/broker/slave/SlaveSynchronize.java | 16 ++-------------- .../rocketmq/srvutil/AclFileWatchService.java | 2 +- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 44bf2a48137..1b29ff173cc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -641,10 +641,7 @@ private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerCo return response; } } - boolean force = false; - if (requestHeader.getForce() != null && requestHeader.getForce()) { - force = true; - } + boolean force = requestHeader.getForce() != null && requestHeader.getForce(); TopicConfig topicConfig = new TopicConfig(topic); topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); @@ -945,13 +942,15 @@ private synchronized RemotingCommand updateColdDataFlowCtrGroupConfig(ChannelHan Properties properties = MixAll.string2Properties(bodyStr); if (properties != null) { LOGGER.info("updateColdDataFlowCtrGroupConfig new config: {}, client: {}", properties, ctx.channel().remoteAddress()); - properties.entrySet().stream().forEach(i -> { + properties.forEach((key, value) -> { try { - String consumerGroup = String.valueOf(i.getKey()); - Long threshold = Long.valueOf(String.valueOf(i.getValue())); - this.brokerController.getColdDataCgCtrService().addOrUpdateGroupConfig(consumerGroup, threshold); + String consumerGroup = String.valueOf(key); + Long threshold = Long.valueOf(String.valueOf(value)); + this.brokerController.getColdDataCgCtrService() + .addOrUpdateGroupConfig(consumerGroup, threshold); } catch (Exception e) { - LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}", i.getKey(), i.getValue(), e); + LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}", + key, value, e); } }); } else { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 7f802adb938..aa77b773ee9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -17,8 +17,6 @@ package org.apache.rocketmq.broker.slave; import java.io.IOException; -import java.util.Iterator; -import java.util.Map; import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang3.StringUtils; @@ -85,12 +83,7 @@ private void syncTopicConfig() { ConcurrentMap newTopicConfigTable = topicWrapper.getTopicConfigTable(); //delete ConcurrentMap topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable(); - for (Iterator> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) { - Map.Entry item = it.next(); - if (!newTopicConfigTable.containsKey(item.getKey())) { - it.remove(); - } - } + topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey())); //update topicConfigTable.putAll(newTopicConfigTable); @@ -104,12 +97,7 @@ private void syncTopicConfig() { ConcurrentMap newTopicConfigTable = topicWrapper.getTopicConfigTable(); //delete ConcurrentMap topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable(); - for (Iterator> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) { - Map.Entry item = it.next(); - if (!newTopicConfigTable.containsKey(item.getKey())) { - it.remove(); - } - } + topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey())); //update topicConfigTable.putAll(newTopicConfigTable); diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java index eff9b422857..9812278d866 100644 --- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/AclFileWatchService.java @@ -73,7 +73,7 @@ public void getAllAclFiles(String path) { return; } File[] files = file.listFiles(); - for (int i = 0; i < files.length; i++) { + for (int i = 0; files != null && i < files.length; i++) { String fileName = files[i].getAbsolutePath(); File f = new File(fileName); if (fileName.equals(aclPath + File.separator + "tools.yml")) { From b3301309327f541c72ade1786f5ea7a3c750811c Mon Sep 17 00:00:00 2001 From: zzl <87265072+zhiliatom@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:55:03 +0800 Subject: [PATCH 042/265] [ISSUE #8291] format proxy watermark output (#8292) --- .../apache/rocketmq/common/thread/ThreadPoolMonitor.java | 7 +++---- proxy/src/main/resources/rmq.proxy.logback.xml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java index 1bfabbffedd..746128d296c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java @@ -105,10 +105,9 @@ public static void logThreadPoolStatus() { List monitors = threadPoolWrapper.getStatusPrinters(); for (ThreadPoolStatusMonitor monitor : monitors) { double value = monitor.value(threadPoolWrapper.getThreadPoolExecutor()); - waterMarkLogger.info("\t{}\t{}\t{}", threadPoolWrapper.getName(), - monitor.describe(), - value); - + String nameFormatted = String.format("%-40s", threadPoolWrapper.getName()); + String descFormatted = String.format("%-12s", monitor.describe()); + waterMarkLogger.info("{}{}{}", nameFormatted, descFormatted, value); if (enablePrintJstack) { if (monitor.needPrintJstack(threadPoolWrapper.getThreadPoolExecutor(), value) && System.currentTimeMillis() - jstackTime > jstackPeriodTime) { diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml index b44b22a6983..f485dcbe3ca 100644 --- a/proxy/src/main/resources/rmq.proxy.logback.xml +++ b/proxy/src/main/resources/rmq.proxy.logback.xml @@ -52,7 +52,7 @@ 128MB - %d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n + %d{yyy-MM-dd HH:mm:ss,GMT+8} %m%n UTF-8 From a89ebe3783c4af610755822ba3a807b5b5dca226 Mon Sep 17 00:00:00 2001 From: yx9o Date: Tue, 25 Jun 2024 12:32:27 +0800 Subject: [PATCH 043/265] [ISSUE #8324] Add more test coverage for DefaultMQProducerImpl (#8325) * [ISSUE #8324] Add more test coverage for DefaultMQProducerImpl * Add license * Update test * Update test * Update test * Update test * Update test * Update test * Update test * Update test --- .../selector/DefaultMQProducerImplTest.java | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java new file mode 100644 index 00000000000..a17fe43f461 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer.selector; + +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; +import org.apache.rocketmq.client.hook.CheckForbiddenContext; +import org.apache.rocketmq.client.hook.CheckForbiddenHook; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.RequestCallback; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.TransactionListener; +import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.common.ServiceState; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.AdditionalMatchers.or; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQProducerImplTest { + + @Mock + private Message message; + + @Mock + private MessageQueue messageQueue; + + @Mock + private MessageQueueSelector queueSelector; + + @Mock + private RequestCallback requestCallback; + + @Mock + private MQClientInstance mQClientFactory; + + private DefaultMQProducerImpl defaultMQProducerImpl; + + private final long defaultTimeout = 30000L; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final String defaultTopic = "testTopic"; + + @Before + public void init() throws Exception { + when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class)); + when(mQClientFactory.getClientId()).thenReturn("client-id"); + when(mQClientFactory.getMQAdminImpl()).thenReturn(mock(MQAdminImpl.class)); + ClientConfig clientConfig = mock(ClientConfig.class); + when(messageQueue.getTopic()).thenReturn(defaultTopic); + when(clientConfig.queueWithNamespace(any())).thenReturn(messageQueue); + when(mQClientFactory.getClientConfig()).thenReturn(clientConfig); + when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class)); + MQClientAPIImpl mQClientAPIImpl = mock(MQClientAPIImpl.class); + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl); + when(mQClientFactory.findBrokerAddressInPublish(or(isNull(), anyString()))).thenReturn(defaultBrokerAddr); + when(message.getTopic()).thenReturn(defaultTopic); + when(message.getProperty(MessageConst.PROPERTY_CORRELATION_ID)).thenReturn("correlation-id"); + when(message.getBody()).thenReturn(new byte[1]); + TransactionMQProducer producer = new TransactionMQProducer("test-producer-group"); + producer.setTransactionListener(mock(TransactionListener.class)); + producer.setTopics(Collections.singletonList(defaultTopic)); + defaultMQProducerImpl = new DefaultMQProducerImpl(producer); + setMQClientFactory(); + setCheckExecutor(); + setCheckForbiddenHookList(); + setTopicPublishInfoTable(); + defaultMQProducerImpl.setServiceState(ServiceState.RUNNING); + } + + @Test + public void testRequest() throws Exception { + defaultMQProducerImpl.request(message, messageQueue, requestCallback, defaultTimeout); + defaultMQProducerImpl.request(message, queueSelector, 1, requestCallback, defaultTimeout); + } + + @Test(expected = MQClientException.class) + public void testRequestMQClientExceptionByVoid() throws Exception { + defaultMQProducerImpl.request(message, requestCallback, defaultTimeout); + } + + @Test + public void testCheckTransactionState() { + defaultMQProducerImpl.checkTransactionState(defaultBrokerAddr, mock(MessageExt.class), mock(CheckTransactionStateRequestHeader.class)); + } + + @Test + public void testCreateTopic() throws MQClientException { + defaultMQProducerImpl.createTopic("key", defaultTopic, 0); + } + + @Test + public void testExecuteCheckForbiddenHook() throws MQClientException { + defaultMQProducerImpl.executeCheckForbiddenHook(mock(CheckForbiddenContext.class)); + } + + @Test(expected = MQClientException.class) + public void testSendOneway() throws MQClientException, InterruptedException, RemotingException { + defaultMQProducerImpl.sendOneway(message); + } + + @Test + public void testSendOnewayByQueueSelector() throws MQClientException, InterruptedException, RemotingException { + defaultMQProducerImpl.sendOneway(message, mock(MessageQueueSelector.class), 1); + } + + @Test + public void testSendOnewayByQueue() throws MQClientException, InterruptedException, RemotingException { + defaultMQProducerImpl.sendOneway(message, messageQueue); + } + + @Test(expected = MQClientException.class) + public void testSend() throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + assertNull(defaultMQProducerImpl.send(message)); + } + + @Test + public void assertSendByQueue() throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + SendResult actual = defaultMQProducerImpl.send(message, messageQueue); + assertNull(actual); + actual = defaultMQProducerImpl.send(message, messageQueue, defaultTimeout); + assertNull(actual); + } + + @Test + public void assertSendByQueueSelector() throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + SendCallback sendCallback = mock(SendCallback.class); + defaultMQProducerImpl.send(message, queueSelector, 1, sendCallback); + SendResult actual = defaultMQProducerImpl.send(message, queueSelector, 1); + assertNull(actual); + actual = defaultMQProducerImpl.send(message, queueSelector, 1, defaultTimeout); + assertNull(actual); + } + + @Test(expected = MQClientException.class) + public void assertMQClientException() throws Exception { + assertNull(defaultMQProducerImpl.request(message, defaultTimeout)); + } + + @Test(expected = RequestTimeoutException.class) + public void assertRequestRequestTimeoutByQueueSelector() throws Exception { + assertNull(defaultMQProducerImpl.request(message, queueSelector, 1, 3000L)); + } + + @Test(expected = Exception.class) + public void assertRequestTimeoutExceptionByQueue() throws Exception { + assertNull(defaultMQProducerImpl.request(message, messageQueue, 3000L)); + } + + @Test + public void testRegisterCheckForbiddenHook() { + CheckForbiddenHook checkForbiddenHook = mock(CheckForbiddenHook.class); + defaultMQProducerImpl.registerCheckForbiddenHook(checkForbiddenHook); + } + + @Test + public void testInitTopicRoute() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Class clazz = defaultMQProducerImpl.getClass(); + Method method = clazz.getDeclaredMethod("initTopicRoute"); + method.setAccessible(true); + method.invoke(defaultMQProducerImpl); + } + + @Test + public void assertFetchPublishMessageQueues() throws MQClientException { + List actual = defaultMQProducerImpl.fetchPublishMessageQueues(defaultTopic); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void assertSearchOffset() throws MQClientException { + assertEquals(0, defaultMQProducerImpl.searchOffset(messageQueue, System.currentTimeMillis())); + } + + @Test + public void assertMaxOffset() throws MQClientException { + assertEquals(0, defaultMQProducerImpl.maxOffset(messageQueue)); + } + + @Test + public void assertMinOffset() throws MQClientException { + assertEquals(0, defaultMQProducerImpl.minOffset(messageQueue)); + } + + @Test + public void assertEarliestMsgStoreTime() throws MQClientException { + assertEquals(0, defaultMQProducerImpl.earliestMsgStoreTime(messageQueue)); + } + + @Test + public void assertViewMessage() throws MQClientException, MQBrokerException, RemotingException, InterruptedException { + assertNull(defaultMQProducerImpl.viewMessage(defaultTopic, "msgId")); + } + + @Test + public void assertQueryMessage() throws MQClientException, InterruptedException { + assertNull(defaultMQProducerImpl.queryMessage(defaultTopic, "key", 1, 0L, 10L)); + } + + @Test + public void assertQueryMessageByUniqKey() throws MQClientException, InterruptedException { + assertNull(defaultMQProducerImpl.queryMessageByUniqKey(defaultTopic, "key")); + } + + @Test + public void assertSetAsyncSenderExecutor() { + ExecutorService asyncSenderExecutor = mock(ExecutorService.class); + defaultMQProducerImpl.setAsyncSenderExecutor(asyncSenderExecutor); + assertEquals(asyncSenderExecutor, defaultMQProducerImpl.getAsyncSenderExecutor()); + } + + @Test + public void assertServiceState() { + ServiceState serviceState = defaultMQProducerImpl.getServiceState(); + assertNotNull(serviceState); + assertEquals(ServiceState.RUNNING, serviceState); + defaultMQProducerImpl.setServiceState(ServiceState.SHUTDOWN_ALREADY); + serviceState = defaultMQProducerImpl.getServiceState(); + assertNotNull(serviceState); + assertEquals(ServiceState.SHUTDOWN_ALREADY, serviceState); + } + + @Test + public void assertGetNotAvailableDuration() { + long[] notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration(); + assertNotNull(notAvailableDuration); + defaultMQProducerImpl.setNotAvailableDuration(new long[1]); + notAvailableDuration = defaultMQProducerImpl.getNotAvailableDuration(); + assertNotNull(notAvailableDuration); + assertEquals(1, notAvailableDuration.length); + } + + @Test + public void assertGetLatencyMax() { + long[] actual = defaultMQProducerImpl.getLatencyMax(); + assertNotNull(actual); + defaultMQProducerImpl.setLatencyMax(new long[1]); + actual = defaultMQProducerImpl.getLatencyMax(); + assertNotNull(actual); + assertEquals(1, actual.length); + } + + @Test + public void assertIsSendLatencyFaultEnable() { + boolean actual = defaultMQProducerImpl.isSendLatencyFaultEnable(); + assertFalse(actual); + defaultMQProducerImpl.setSendLatencyFaultEnable(true); + actual = defaultMQProducerImpl.isSendLatencyFaultEnable(); + assertTrue(actual); + } + + @Test + public void assertGetMqFaultStrategy() { + assertNotNull(defaultMQProducerImpl.getMqFaultStrategy()); + } + + @Test + public void assertCheckListener() { + assertNull(defaultMQProducerImpl.checkListener()); + } + + private void setMQClientFactory() throws IllegalAccessException, NoSuchFieldException { + setField(defaultMQProducerImpl, "mQClientFactory", mQClientFactory); + } + + private void setTopicPublishInfoTable() throws IllegalAccessException, NoSuchFieldException { + ConcurrentMap topicPublishInfoTable = new ConcurrentHashMap<>(); + TopicPublishInfo topicPublishInfo = mock(TopicPublishInfo.class); + when(topicPublishInfo.ok()).thenReturn(true); + topicPublishInfoTable.put(defaultTopic, topicPublishInfo); + setField(defaultMQProducerImpl, "topicPublishInfoTable", topicPublishInfoTable); + } + + private void setCheckExecutor() throws NoSuchFieldException, IllegalAccessException { + setField(defaultMQProducerImpl, "checkExecutor", mock(ExecutorService.class)); + } + + private void setCheckForbiddenHookList() throws NoSuchFieldException, IllegalAccessException { + ArrayList checkForbiddenHookList = new ArrayList<>(); + checkForbiddenHookList.add(mock(CheckForbiddenHook.class)); + setField(defaultMQProducerImpl, "checkForbiddenHookList", checkForbiddenHookList); + } + + private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException { + Class clazz = target.getClass(); + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, newValue); + } +} From b31e0cea00a8d95921877fd8c1e4239717b05a9e Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan Date: Wed, 26 Jun 2024 19:04:45 +0800 Subject: [PATCH 044/265] =?UTF-8?q?Revert=20"[ISSUE=20#7686]=20The=20bornT?= =?UTF-8?q?ime=20is=20not=20set=20when=20using=20the=20popMessage=20API=20?= =?UTF-8?q?i=E2=80=A6"=20(#8331)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 4bb4d78f1d5a8d920b85675ef9628a75b2a86f98. --- .../org/apache/rocketmq/proxy/processor/ConsumerProcessor.java | 1 - .../rocketmq/proxy/service/message/LocalMessageService.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java index 9adf20ebba2..24fc0a2a28f 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java @@ -137,7 +137,6 @@ public CompletableFuture popMessage( requestHeader.setExp(subscriptionData.getSubString()); requestHeader.setOrder(fifo); requestHeader.setAttemptId(attemptId); - requestHeader.setBornTime(System.currentTimeMillis()); future = this.serviceManager.getMessageService().popMessage( ctx, diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index 9181f966f4c..aaa688fee64 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -195,6 +195,7 @@ public CompletableFuture endTransactionOneway(ProxyContext ctx, String bro @Override public CompletableFuture popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, PopMessageRequestHeader requestHeader, long timeoutMillis) { + requestHeader.setBornTime(System.currentTimeMillis()); RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader, ctx.getLanguage()); CompletableFuture future = new CompletableFuture<>(); SimpleChannel channel = channelManager.createInvocationChannel(ctx); From c6d3f2615bf7e2d63b1d6baf8a0d94b1cff05830 Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 1 Jul 2024 13:17:07 +0800 Subject: [PATCH 045/265] [ISSUE #8343] Add more test coverage for MQClientAPIImpl (#8344) --- .../client/impl/MQClientAPIImplTest.java | 1849 +++++++++++++---- 1 file changed, 1416 insertions(+), 433 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index b0876c7c0d9..e311e0c9b85 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -16,13 +16,6 @@ */ package org.apache.rocketmq.client.impl; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.AckCallback; import org.apache.rocketmq.client.consumer.AckResult; @@ -30,6 +23,9 @@ import org.apache.rocketmq.client.consumer.PopCallback; import org.apache.rocketmq.client.consumer.PopResult; import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.hook.SendMessageContext; @@ -39,8 +35,10 @@ import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -49,20 +47,65 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueAssignment; import org.apache.rocketmq.common.message.MessageRequestMode; +import org.apache.rocketmq.common.namesrv.TopAddressing; +import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.common.HeartbeatV2Result; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.AclInfo; +import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; +import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; +import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; +import org.apache.rocketmq.remoting.protocol.body.BrokerStatsItem; +import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.Connection; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData; +import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; +import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; +import org.apache.rocketmq.remoting.protocol.body.KVTable; +import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.remoting.protocol.body.ProducerConnection; +import org.apache.rocketmq.remoting.protocol.body.ProducerInfo; +import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo; import org.apache.rocketmq.remoting.protocol.body.QueryAssignmentResponseBody; +import org.apache.rocketmq.remoting.protocol.body.QueryConsumeQueueResponseBody; +import org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody; +import org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody; +import org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.remoting.protocol.body.UserInfo; import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.remoting.protocol.header.GetBrokerAclConfigResponseHeader; import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseBody; import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupResponseHeader; import org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader; @@ -70,15 +113,33 @@ import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.namesrv.GetKVConfigResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; +import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; +import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.assertj.core.api.Assertions; import org.junit.Before; @@ -86,34 +147,74 @@ import org.junit.runner.RunWith; import org.mockito.ArgumentMatchers; import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class MQClientAPIImplTest { + private MQClientAPIImpl mqClientAPI = new MQClientAPIImpl(new NettyClientConfig(), null, null, new ClientConfig()); + @Mock private RemotingClient remotingClient; + @Mock private DefaultMQProducerImpl defaultMQProducerImpl; - private String brokerAddr = "127.0.0.1"; - private String brokerName = "DefaultBroker"; - private String clusterName = "DefaultCluster"; - private static String group = "FooBarGroup"; - private static String topic = "FooBar"; - private Message msg = new Message("FooBar", new byte[] {}); - private static String clientId = "127.0.0.2@UnitTest"; + @Mock + private RemotingCommand response; + + private final String brokerAddr = "127.0.0.1"; + + private final String brokerName = "DefaultBroker"; + + private final String clusterName = "DefaultCluster"; + + private final String group = "FooBarGroup"; + + private final String topic = "FooBar"; + + private final Message msg = new Message("FooBar", new byte[]{}); + + private final String clientId = "127.0.0.2@UnitTest"; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final String defaultNsAddr = "127.0.0.1:9876"; + + private final long defaultTimeout = 3000L; @Before public void init() throws Exception { @@ -153,12 +254,9 @@ public void testSendMessageOneWay_WithException() throws RemotingException, Inte @Test public void testSendMessageSync_Success() throws InterruptedException, RemotingException, MQBrokerException { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createSendMessageSuccessResponse(request); - } + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createSendMessageSuccessResponse(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); SendMessageRequestHeader requestHeader = createSendMessageRequestHeader(); @@ -173,17 +271,14 @@ public Object answer(InvocationOnMock mock) throws Throwable { } @Test - public void testSendMessageSync_WithException() throws InterruptedException, RemotingException, MQBrokerException { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setOpaque(request.getOpaque()); - response.setRemark("Broker is broken."); - return response; - } + public void testSendMessageSync_WithException() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setOpaque(request.getOpaque()); + response.setRemark("Broker is broken."); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); SendMessageRequestHeader requestHeader = createSendMessageRequestHeader(); @@ -204,16 +299,13 @@ public void testSendMessageAsync_Success() throws RemotingException, Interrupted 3 * 1000, CommunicationMode.ASYNC, new SendMessageContext(), defaultMQProducerImpl); assertThat(sendResult).isNull(); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer(mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); SendMessageContext sendMessageContext = new SendMessageContext(); sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer())); @@ -266,34 +358,26 @@ public void onException(Throwable e) { } @Test - public void testCreatePlainAccessConfig_Success() throws InterruptedException, RemotingException, MQBrokerException { - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createSuccessResponse4UpdateAclConfig(request); - } + public void testCreatePlainAccessConfig_Success() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createSuccessResponse4UpdateAclConfig(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); PlainAccessConfig config = createUpdateAclConfig(); try { mqClientAPI.createPlainAccessConfig(brokerAddr, config, 3 * 1000); - } catch (MQClientException ex) { + } catch (MQClientException ignored) { } } @Test - public void testCreatePlainAccessConfig_Exception() throws InterruptedException, RemotingException, MQBrokerException { - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createErrorResponse4UpdateAclConfig(request); - } + public void testCreatePlainAccessConfig_Exception() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createErrorResponse4UpdateAclConfig(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); PlainAccessConfig config = createUpdateAclConfig(); @@ -306,33 +390,25 @@ public Object answer(InvocationOnMock mock) throws Throwable { } @Test - public void testDeleteAccessConfig_Success() throws InterruptedException, RemotingException, MQBrokerException { - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createSuccessResponse4DeleteAclConfig(request); - } + public void testDeleteAccessConfig_Success() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createSuccessResponse4DeleteAclConfig(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); String accessKey = "1234567"; try { mqClientAPI.deleteAccessConfig(brokerAddr, accessKey, 3 * 1000); - } catch (MQClientException ex) { + } catch (MQClientException ignored) { } } @Test - public void testDeleteAccessConfig_Exception() throws InterruptedException, RemotingException, MQBrokerException { - - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createErrorResponse4DeleteAclConfig(request); - } + public void testDeleteAccessConfig_Exception() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createErrorResponse4DeleteAclConfig(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); try { @@ -344,17 +420,14 @@ public Object answer(InvocationOnMock mock) throws Throwable { } @Test - public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setOpaque(request.getOpaque()); - response.setRemark("Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed."); - return response; - } + public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setOpaque(request.getOpaque()); + response.setRemark("Put message back to RMQ_SYS_TRANS_HALF_TOPIC failed."); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "topic,", "test", 3000); @@ -362,13 +435,10 @@ public Object answer(InvocationOnMock mock) throws Throwable { } @Test - public void testResumeCheckHalfMessage_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - RemotingCommand request = mock.getArgument(1); - return createResumeSuccessResponse(request); - } + public void testResumeCheckHalfMessage_Success() throws InterruptedException, RemotingException { + doAnswer(mock -> { + RemotingCommand request = mock.getArgument(1); + return createResumeSuccessResponse(request); }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); boolean result = mqClientAPI.resumeCheckHalfMessage(brokerAddr, "topic", "test", 3000); @@ -378,16 +448,13 @@ public Object answer(InvocationOnMock mock) throws Throwable { @Test public void testSendMessageTypeofReply() throws Exception { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer(mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(ArgumentMatchers.anyString(), ArgumentMatchers.any(RemotingCommand.class), ArgumentMatchers.anyLong(), ArgumentMatchers.any(InvokeCallback.class)); SendMessageContext sendMessageContext = new SendMessageContext(); sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer())); @@ -410,19 +477,16 @@ public void onException(Throwable e) { @Test public void testQueryAssignment_Success() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); - - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - QueryAssignmentResponseBody b = new QueryAssignmentResponseBody(); - b.setMessageQueueAssignments(Collections.singleton(new MessageQueueAssignment())); - response.setBody(b.encode()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + QueryAssignmentResponseBody b = new QueryAssignmentResponseBody(); + b.setMessageQueueAssignments(Collections.singleton(new MessageQueueAssignment())); + response.setBody(b.encode()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); Set assignments = mqClientAPI.queryAssignment(brokerAddr, topic, group, clientId, null, MessageModel.CLUSTERING, 10 * 1000); assertThat(assignments).size().isEqualTo(1); @@ -432,48 +496,45 @@ public RemotingCommand answer(InvocationOnMock mock) { public void testPopMessageAsync_Success() throws Exception { final long popTime = System.currentTimeMillis(); final int invisibleTime = 10 * 1000; - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - - PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); - responseHeader.setInvisibleTime(invisibleTime); - responseHeader.setPopTime(popTime); - responseHeader.setReviveQid(0); - responseHeader.setRestNum(1); - StringBuilder startOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); - responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); - StringBuilder msgOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); - responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); - response.setRemark("FOUND"); - response.makeCustomHeaderToNet(); - - MessageExt message = new MessageExt(); - message.setQueueId(0); - message.setFlag(12); - message.setQueueOffset(0L); - message.setCommitLogOffset(100L); - message.setSysFlag(0); - message.setBornTimestamp(System.currentTimeMillis()); - message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); - message.setStoreTimestamp(System.currentTimeMillis()); - message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); - message.setBody("body".getBytes()); - message.setTopic(topic); - message.putUserProperty("key", "value"); - response.setBody(MessageDecoder.encode(message, false)); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + + PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + responseHeader.setInvisibleTime(invisibleTime); + responseHeader.setPopTime(popTime); + responseHeader.setReviveQid(0); + responseHeader.setRestNum(1); + StringBuilder startOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); + responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); + StringBuilder msgOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); + responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); + response.setRemark("FOUND"); + response.makeCustomHeaderToNet(); + + MessageExt message = new MessageExt(); + message.setQueueId(0); + message.setFlag(12); + message.setQueueOffset(0L); + message.setCommitLogOffset(100L); + message.setSysFlag(0); + message.setBornTimestamp(System.currentTimeMillis()); + message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); + message.setStoreTimestamp(System.currentTimeMillis()); + message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); + message.setBody("body".getBytes()); + message.setTopic(topic); + message.putUserProperty("key", "value"); + response.setBody(MessageDecoder.encode(message, false)); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); final CountDownLatch done = new CountDownLatch(1); mqClientAPI.popMessageAsync(brokerName, brokerAddr, new PopMessageRequestHeader(), 10 * 1000, new PopCallback() { @@ -501,50 +562,47 @@ public void testPopLmqMessage_async() throws Exception { final long popTime = System.currentTimeMillis(); final int invisibleTime = 10 * 1000; final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1"; - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - - PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); - responseHeader.setInvisibleTime(invisibleTime); - responseHeader.setPopTime(popTime); - responseHeader.setReviveQid(0); - responseHeader.setRestNum(1); - StringBuilder startOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); - responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); - StringBuilder msgOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); - responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); - response.setRemark("FOUND"); - response.makeCustomHeaderToNet(); - - MessageExt message = new MessageExt(); - message.setQueueId(3); - message.setFlag(0); - message.setQueueOffset(5L); - message.setCommitLogOffset(11111L); - message.setSysFlag(0); - message.setBornTimestamp(System.currentTimeMillis()); - message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); - message.setStoreTimestamp(System.currentTimeMillis()); - message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); - message.setBody("body".getBytes()); - message.setTopic(topic); - message.putUserProperty("key", "value"); - message.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic); - message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); - response.setBody(MessageDecoder.encode(message, false)); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + + PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + responseHeader.setInvisibleTime(invisibleTime); + responseHeader.setPopTime(popTime); + responseHeader.setReviveQid(0); + responseHeader.setRestNum(1); + StringBuilder startOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); + responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); + StringBuilder msgOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); + responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); + response.setRemark("FOUND"); + response.makeCustomHeaderToNet(); + + MessageExt message = new MessageExt(); + message.setQueueId(3); + message.setFlag(0); + message.setQueueOffset(5L); + message.setCommitLogOffset(11111L); + message.setSysFlag(0); + message.setBornTimestamp(System.currentTimeMillis()); + message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); + message.setStoreTimestamp(System.currentTimeMillis()); + message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); + message.setBody("body".getBytes()); + message.setTopic(topic); + message.putUserProperty("key", "value"); + message.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, lmqTopic); + message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); + response.setBody(MessageDecoder.encode(message, false)); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); final CountDownLatch done = new CountDownLatch(1); final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); @@ -580,49 +638,46 @@ public void testPopMultiLmqMessage_async() throws Exception { final String lmqTopic2 = MixAll.LMQ_PREFIX + "lmq2"; final String multiDispatch = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, lmqTopic, lmqTopic2); final String multiOffset = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, "0", "0"); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - - PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); - responseHeader.setInvisibleTime(invisibleTime); - responseHeader.setPopTime(popTime); - responseHeader.setReviveQid(0); - responseHeader.setRestNum(1); - StringBuilder startOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); - responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); - StringBuilder msgOffsetInfo = new StringBuilder(64); - ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); - responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); - response.setRemark("FOUND"); - response.makeCustomHeaderToNet(); - - MessageExt message = new MessageExt(); - message.setQueueId(0); - message.setFlag(0); - message.setQueueOffset(10L); - message.setCommitLogOffset(10000L); - message.setSysFlag(0); - message.setBornTimestamp(System.currentTimeMillis()); - message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); - message.setStoreTimestamp(System.currentTimeMillis()); - message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); - message.setBody("body".getBytes()); - message.setTopic(topic); - MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, multiDispatch); - MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, multiOffset); - response.setBody(MessageDecoder.encode(message, false)); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + + PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + responseHeader.setInvisibleTime(invisibleTime); + responseHeader.setPopTime(popTime); + responseHeader.setReviveQid(0); + responseHeader.setRestNum(1); + StringBuilder startOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, topic, 0, 0L); + responseHeader.setStartOffsetInfo(startOffsetInfo.toString()); + StringBuilder msgOffsetInfo = new StringBuilder(64); + ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, topic, 0, Collections.singletonList(0L)); + responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString()); + response.setRemark("FOUND"); + response.makeCustomHeaderToNet(); + + MessageExt message = new MessageExt(); + message.setQueueId(0); + message.setFlag(0); + message.setQueueOffset(10L); + message.setCommitLogOffset(10000L); + message.setSysFlag(0); + message.setBornTimestamp(System.currentTimeMillis()); + message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); + message.setStoreTimestamp(System.currentTimeMillis()); + message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); + message.setBody("body".getBytes()); + message.setTopic(topic); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_DISPATCH, multiDispatch); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, multiOffset); + response.setBody(MessageDecoder.encode(message, false)); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); final CountDownLatch done = new CountDownLatch(1); final PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); @@ -654,19 +709,16 @@ public void onException(Throwable e) { @Test public void testAckMessageAsync_Success() throws Exception { - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); - response.setOpaque(request.getOpaque()); - response.setCode(ResponseCode.SUCCESS); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); final CountDownLatch done = new CountDownLatch(1); @@ -688,22 +740,19 @@ public void onException(Throwable e) { @Test public void testChangeInvisibleTimeAsync_Success() throws Exception { - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock mock) throws Throwable { - InvokeCallback callback = mock.getArgument(3); - RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); - response.setOpaque(request.getOpaque()); - response.setCode(ResponseCode.SUCCESS); - ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader(); - responseHeader.setPopTime(System.currentTimeMillis()); - responseHeader.setInvisibleTime(10 * 1000L); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - } + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader(); + responseHeader.setPopTime(System.currentTimeMillis()); + responseHeader.setInvisibleTime(10 * 1000L); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); final CountDownLatch done = new CountDownLatch(1); @@ -730,16 +779,13 @@ public void onException(Throwable e) { @Test public void testSetMessageRequestMode_Success() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); mqClientAPI.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 10 * 1000L); @@ -747,16 +793,13 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testCreateSubscriptionGroup_Success() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); mqClientAPI.createSubscriptionGroup(brokerAddr, new SubscriptionGroupConfig(), 10000); @@ -764,16 +807,13 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testCreateTopic_Success() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); mqClientAPI.createTopic(brokerAddr, topic, new TopicConfig(), 10000); @@ -781,31 +821,28 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testViewMessage() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) throws Exception { - RemotingCommand request = mock.getArgument(1); - - RemotingCommand response = RemotingCommand.createResponseCommand(null); - MessageExt message = new MessageExt(); - message.setQueueId(0); - message.setFlag(12); - message.setQueueOffset(0L); - message.setCommitLogOffset(100L); - message.setSysFlag(0); - message.setBornTimestamp(System.currentTimeMillis()); - message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); - message.setStoreTimestamp(System.currentTimeMillis()); - message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); - message.setBody("body".getBytes()); - message.setTopic(topic); - message.putUserProperty("key", "value"); - response.setBody(MessageDecoder.encode(message, false)); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + MessageExt message = new MessageExt(); + message.setQueueId(0); + message.setFlag(12); + message.setQueueOffset(0L); + message.setCommitLogOffset(100L); + message.setSysFlag(0); + message.setBornTimestamp(System.currentTimeMillis()); + message.setBornHost(new InetSocketAddress("127.0.0.1", 10)); + message.setStoreTimestamp(System.currentTimeMillis()); + message.setStoreHost(new InetSocketAddress("127.0.0.1", 11)); + message.setBody("body".getBytes()); + message.setTopic(topic); + message.putUserProperty("key", "value"); + response.setBody(MessageDecoder.encode(message, false)); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); MessageExt messageExt = mqClientAPI.viewMessage(brokerAddr, "topic", 100L, 10000); @@ -814,19 +851,16 @@ public RemotingCommand answer(InvocationOnMock mock) throws Exception { @Test public void testSearchOffset() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); - - final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); - final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(100L); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(100L); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); long offset = mqClientAPI.searchOffset(brokerAddr, topic, 0, System.currentTimeMillis() - 1000, 10000); @@ -835,19 +869,16 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testGetMaxOffset() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); - - final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); - final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(100L); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); + final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(100L); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); long offset = mqClientAPI.getMaxOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000); @@ -856,19 +887,16 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testGetMinOffset() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); - - final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class); - final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(100L); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class); + final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(100L); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); long offset = mqClientAPI.getMinOffset(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000); @@ -877,19 +905,16 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testGetEarliestMsgStoretime() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); - - final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class); - final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader(); - responseHeader.setTimestamp(100L); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class); + final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader(); + responseHeader.setTimestamp(100L); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); long t = mqClientAPI.getEarliestMsgStoretime(brokerAddr, new MessageQueue(topic, brokerName, 0), 10000); @@ -898,21 +923,18 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testQueryConsumerOffset() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - final RemotingCommand response = + final RemotingCommand response = RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class); - final QueryConsumerOffsetResponseHeader responseHeader = + final QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(100L); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + responseHeader.setOffset(100L); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); long t = mqClientAPI.queryConsumerOffset(brokerAddr, new QueryConsumerOffsetRequestHeader(), 1000); @@ -921,18 +943,15 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testUpdateConsumerOffset() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - final RemotingCommand response = + final RemotingCommand response = RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); mqClientAPI.updateConsumerOffset(brokerAddr, new UpdateConsumerOffsetRequestHeader(), 1000); @@ -940,21 +959,18 @@ public RemotingCommand answer(InvocationOnMock mock) { @Test public void testGetConsumerIdListByGroup() throws Exception { - doAnswer(new Answer() { - @Override - public RemotingCommand answer(InvocationOnMock mock) { - RemotingCommand request = mock.getArgument(1); + doAnswer((Answer) mock -> { + RemotingCommand request = mock.getArgument(1); - final RemotingCommand response = + final RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class); - GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody(); - body.setConsumerIdList(Collections.singletonList("consumer1")); - response.setBody(body.encode()); - response.makeCustomHeaderToNet(); - response.setCode(ResponseCode.SUCCESS); - response.setOpaque(request.getOpaque()); - return response; - } + GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody(); + body.setConsumerIdList(Collections.singletonList("consumer1")); + response.setBody(body.encode()); + response.makeCustomHeaderToNet(); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); List consumerIdList = mqClientAPI.getConsumerIdListByGroup(brokerAddr, group, 10000); assertThat(consumerIdList).size().isGreaterThan(0); @@ -991,7 +1007,6 @@ private RemotingCommand createSuccessResponse4UpdateAclConfig(RemotingCommand re response.setOpaque(request.getOpaque()); response.markResponseType(); response.setRemark(null); - return response; } @@ -1001,7 +1016,6 @@ private RemotingCommand createSuccessResponse4DeleteAclConfig(RemotingCommand re response.setOpaque(request.getOpaque()); response.markResponseType(); response.setRemark(null); - return response; } @@ -1011,7 +1025,6 @@ private RemotingCommand createErrorResponse4UpdateAclConfig(RemotingCommand requ response.setOpaque(request.getOpaque()); response.markResponseType(); response.setRemark("corresponding to accessConfig has been updated failed"); - return response; } @@ -1021,12 +1034,10 @@ private RemotingCommand createErrorResponse4DeleteAclConfig(RemotingCommand requ response.setOpaque(request.getOpaque()); response.markResponseType(); response.setRemark("corresponding to accessConfig has been deleted failed"); - return response; } private PlainAccessConfig createUpdateAclConfig() { - PlainAccessConfig config = new PlainAccessConfig(); config.setAccessKey("Rocketmq111"); config.setSecretKey("123456789"); @@ -1049,21 +1060,18 @@ private SendMessageRequestHeader createSendMessageRequestHeader() { @Test public void testAddWritePermOfBroker() throws Exception { - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - RemotingCommand request = invocationOnMock.getArgument(1); - if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) { - return null; - } - - RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class); - AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader(); - response.setCode(ResponseCode.SUCCESS); - responseHeader.setAddTopicCount(7); - response.addExtField("addTopicCount", String.valueOf(responseHeader.getAddTopicCount())); - return response; + doAnswer(invocationOnMock -> { + RemotingCommand request = invocationOnMock.getArgument(1); + if (request.getCode() != RequestCode.ADD_WRITE_PERM_OF_BROKER) { + return null; } + + RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class); + AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader(); + response.setCode(ResponseCode.SUCCESS); + responseHeader.setAddTopicCount(7); + response.addExtField("addTopicCount", String.valueOf(responseHeader.getAddTopicCount())); + return response; }).when(remotingClient).invokeSync(anyString(), any(RemotingCommand.class), anyLong()); int topicCnt = mqClientAPI.addWritePermOfBroker("127.0.0.1", "default-broker", 1000); @@ -1091,4 +1099,979 @@ public void testCreateTopicList_Success() throws RemotingException, InterruptedE mqClientAPI.createTopicList(brokerAddr, topicConfigList, 10000); } + @Test + public void assertFetchNameServerAddr() throws NoSuchFieldException, IllegalAccessException { + setTopAddressing(); + assertEquals(defaultNsAddr, mqClientAPI.fetchNameServerAddr()); + } + + @Test + public void assertOnNameServerAddressChange() { + assertEquals(defaultNsAddr, mqClientAPI.onNameServerAddressChange(defaultNsAddr)); + } + + @Test(expected = AssertionError.class) + public void testUpdateGlobalWhiteAddrsConfig() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + mqClientAPI.updateGlobalWhiteAddrsConfig(defaultNsAddr, "", "", defaultTimeout); + } + + @Test + public void assertGetBrokerClusterAclInfo() throws MQBrokerException, RemotingException, InterruptedException { + mockInvokeSync(); + GetBrokerAclConfigResponseHeader responseHeader = mock(GetBrokerAclConfigResponseHeader.class); + when(responseHeader.getBrokerName()).thenReturn(brokerName); + when(responseHeader.getBrokerAddr()).thenReturn(defaultBrokerAddr); + when(responseHeader.getClusterName()).thenReturn(clusterName); + when(responseHeader.getAllAclFileVersion()).thenReturn("{\"key\":{\"stateVersion\":1}}"); + setResponseHeader(responseHeader); + ClusterAclVersionInfo actual = mqClientAPI.getBrokerClusterAclInfo(defaultNsAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(brokerName, actual.getBrokerName()); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + assertEquals(clusterName, actual.getClusterName()); + assertEquals(1, actual.getAllAclConfigDataVersion().size()); + assertNull(actual.getAclConfigDataVersion()); + } + + @Test + public void assertPullMessage() throws MQBrokerException, RemotingException, InterruptedException { + PullMessageRequestHeader requestHeader = mock(PullMessageRequestHeader.class); + mockInvokeSync(); + PullCallback callback = mock(PullCallback.class); + PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class); + setResponseHeader(responseHeader); + when(responseHeader.getNextBeginOffset()).thenReturn(1L); + when(responseHeader.getMinOffset()).thenReturn(1L); + when(responseHeader.getMaxOffset()).thenReturn(10L); + when(responseHeader.getSuggestWhichBrokerId()).thenReturn(MixAll.MASTER_ID); + PullResult actual = mqClientAPI.pullMessage(defaultBrokerAddr, requestHeader, defaultTimeout, CommunicationMode.SYNC, callback); + assertNotNull(actual); + assertEquals(1L, actual.getNextBeginOffset()); + assertEquals(1L, actual.getMinOffset()); + assertEquals(10L, actual.getMaxOffset()); + assertEquals(PullStatus.FOUND, actual.getPullStatus()); + assertNull(actual.getMsgFoundList()); + } + + @Test + public void testBatchAckMessageAsync() throws MQBrokerException, RemotingException, InterruptedException { + AckCallback callback = mock(AckCallback.class); + List extraInfoList = new ArrayList<>(); + extraInfoList.add(String.format("%s %s %s %s %s %s %d %d", "1", "2", "3", "4", "5", brokerName, 7, 8)); + mqClientAPI.batchAckMessageAsync(defaultBrokerAddr, defaultTimeout, callback, defaultTopic, "", extraInfoList); + } + + @Test + public void assertSearchOffset() throws MQBrokerException, RemotingException, InterruptedException { + mockInvokeSync(); + SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class); + when(responseHeader.getOffset()).thenReturn(1L); + setResponseHeader(responseHeader); + assertEquals(1L, mqClientAPI.searchOffset(defaultBrokerAddr, new MessageQueue(), System.currentTimeMillis(), defaultTimeout)); + } + + @Test + public void testUpdateConsumerOffsetOneway() throws RemotingException, InterruptedException { + UpdateConsumerOffsetRequestHeader requestHeader = mock(UpdateConsumerOffsetRequestHeader.class); + mqClientAPI.updateConsumerOffsetOneway(defaultBrokerAddr, requestHeader, defaultTimeout); + } + + @Test + public void assertSendHeartbeat() throws MQBrokerException, RemotingException, InterruptedException { + mockInvokeSync(); + HeartbeatData heartbeatData = new HeartbeatData(); + assertEquals(1, mqClientAPI.sendHeartbeat(defaultBrokerAddr, heartbeatData, defaultTimeout)); + } + + @Test + public void assertSendHeartbeatV2() throws MQBrokerException, RemotingException, InterruptedException { + mockInvokeSync(); + HeartbeatData heartbeatData = new HeartbeatData(); + HeartbeatV2Result actual = mqClientAPI.sendHeartbeatV2(defaultBrokerAddr, heartbeatData, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getVersion()); + assertFalse(actual.isSubChange()); + assertFalse(actual.isSupportV2()); + } + + @Test + public void testUnregisterClient() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.unregisterClient(defaultBrokerAddr, "", "", "", defaultTimeout); + } + + @Test + public void testEndTransactionOneway() throws RemotingException, InterruptedException { + mockInvokeSync(); + EndTransactionRequestHeader requestHeader = mock(EndTransactionRequestHeader.class); + mqClientAPI.endTransactionOneway(defaultBrokerAddr, requestHeader, "", defaultTimeout); + } + + @Test + public void testQueryMessage() throws MQBrokerException, RemotingException, InterruptedException { + QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class); + InvokeCallback callback = mock(InvokeCallback.class); + mqClientAPI.queryMessage(defaultBrokerAddr, requestHeader, defaultTimeout, callback, false); + } + + @Test + public void testRegisterClient() throws RemotingException, InterruptedException { + mockInvokeSync(); + HeartbeatData heartbeatData = new HeartbeatData(); + assertTrue(mqClientAPI.registerClient(defaultBrokerAddr, heartbeatData, defaultTimeout)); + } + + @Test + public void testConsumerSendMessageBack() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + MessageExt messageExt = mock(MessageExt.class); + mqClientAPI.consumerSendMessageBack(defaultBrokerAddr, brokerName, messageExt, "", 1, defaultTimeout, 1000); + } + + @Test + public void assertLockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + LockBatchRequestBody responseBody = new LockBatchRequestBody(); + setResponseBody(responseBody); + Set actual = mqClientAPI.lockBatchMQ(defaultBrokerAddr, responseBody, defaultTimeout); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testUnlockBatchMQ() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + UnlockBatchRequestBody unlockBatchRequestBody = new UnlockBatchRequestBody(); + mqClientAPI.unlockBatchMQ(defaultBrokerAddr, unlockBatchRequestBody, defaultTimeout, false); + } + + @Test + public void assertGetTopicStatsInfo() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + TopicStatsTable responseBody = new TopicStatsTable(); + MessageQueue messageQueue = new MessageQueue(); + TopicOffset topicOffset = new TopicOffset(); + responseBody.getOffsetTable().put(messageQueue, topicOffset); + setResponseBody(responseBody); + TopicStatsTable actual = mqClientAPI.getTopicStatsInfo(defaultBrokerAddr, defaultTopic, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getOffsetTable().size()); + } + + @Test + public void assertGetConsumeStats() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + ConsumeStats responseBody = new ConsumeStats(); + responseBody.setConsumeTps(1000); + setResponseBody(responseBody); + ConsumeStats actual = mqClientAPI.getConsumeStats(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals(1000, actual.getConsumeTps(), 0.0); + } + + @Test + public void assertGetProducerConnectionList() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + ProducerConnection responseBody = new ProducerConnection(); + responseBody.getConnectionSet().add(new Connection()); + setResponseBody(responseBody); + ProducerConnection actual = mqClientAPI.getProducerConnectionList(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getConnectionSet().size()); + } + + @Test + public void assertGetAllProducerInfo() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + Map> data = new HashMap<>(); + data.put("key", Collections.emptyList()); + ProducerTableInfo responseBody = new ProducerTableInfo(data); + setResponseBody(responseBody); + ProducerTableInfo actual = mqClientAPI.getAllProducerInfo(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getData().size()); + } + + @Test + public void assertGetConsumerConnectionList() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + ConsumerConnection responseBody = new ConsumerConnection(); + responseBody.setConsumeType(ConsumeType.CONSUME_ACTIVELY); + responseBody.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + responseBody.setMessageModel(MessageModel.CLUSTERING); + setResponseBody(responseBody); + ConsumerConnection actual = mqClientAPI.getConsumerConnectionList(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals(ConsumeType.CONSUME_ACTIVELY, actual.getConsumeType()); + assertEquals(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, actual.getConsumeFromWhere()); + assertEquals(MessageModel.CLUSTERING, actual.getMessageModel()); + } + + @Test + public void assertGetBrokerRuntimeInfo() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + KVTable responseBody = new KVTable(); + responseBody.getTable().put("key", "value"); + setResponseBody(responseBody); + KVTable actual = mqClientAPI.getBrokerRuntimeInfo(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTable().size()); + } + + @Test + public void testAddBroker() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.addBroker(defaultBrokerAddr, "", defaultTimeout); + } + + @Test + public void testRemoveBroker() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.removeBroker(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID, defaultTimeout); + } + + @Test + public void testUpdateBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException, MQClientException { + mockInvokeSync(); + mqClientAPI.updateBrokerConfig(defaultBrokerAddr, createProperties(), defaultTimeout); + } + + @Test + public void assertGetBrokerConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException { + mockInvokeSync(); + setResponseBody("{\"key\":\"value\"}"); + Properties actual = mqClientAPI.getBrokerConfig(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + } + + @Test + public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException { + mockInvokeSync(); + Properties props = new Properties(); + mqClientAPI.updateColdDataFlowCtrGroupConfig(defaultBrokerAddr, props, defaultTimeout); + } + + @Test + public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException { + mockInvokeSync(); + mqClientAPI.removeColdDataFlowCtrGroupConfig(defaultBrokerAddr, "", defaultTimeout); + } + + @Test + public void assertGetColdDataFlowCtrInfo() throws RemotingException, InterruptedException, MQBrokerException, UnsupportedEncodingException { + mockInvokeSync(); + setResponseBody("{\"key\":\"value\"}"); + String actual = mqClientAPI.getColdDataFlowCtrInfo(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals("\"{\\\"key\\\":\\\"value\\\"}\"", actual); + } + + @Test + public void assertSetCommitLogReadAheadMode() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + when(response.getRemark()).thenReturn("remark"); + String actual = mqClientAPI.setCommitLogReadAheadMode(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals("remark", actual); + } + + @Test + public void assertGetBrokerClusterInfo() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + ClusterInfo responseBody = new ClusterInfo(); + Map> clusterAddrTable = new HashMap<>(); + clusterAddrTable.put(clusterName, new HashSet<>()); + Map brokerAddrTable = new HashMap<>(); + brokerAddrTable.put(brokerName, new BrokerData()); + responseBody.setClusterAddrTable(clusterAddrTable); + responseBody.setBrokerAddrTable(brokerAddrTable); + setResponseBody(responseBody); + ClusterInfo actual = mqClientAPI.getBrokerClusterInfo(defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getClusterAddrTable().size()); + assertEquals(1, actual.getBrokerAddrTable().size()); + } + + @Test + public void assertGetDefaultTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicRouteData responseBody = new TopicRouteData(); + responseBody.getQueueDatas().add(new QueueData()); + responseBody.getBrokerDatas().add(new BrokerData()); + responseBody.getFilterServerTable().put("key", Collections.emptyList()); + Map topicQueueMappingByBroker = new HashMap<>(); + topicQueueMappingByBroker.put("key", new TopicQueueMappingInfo()); + responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker); + setResponseBody(responseBody); + TopicRouteData actual = mqClientAPI.getDefaultTopicRouteInfoFromNameServer(defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getQueueDatas().size()); + assertEquals(1, actual.getBrokerDatas().size()); + assertEquals(1, actual.getFilterServerTable().size()); + assertEquals(1, actual.getTopicQueueMappingByBroker().size()); + } + + @Test + public void assertGetTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicRouteData responseBody = new TopicRouteData(); + responseBody.getQueueDatas().add(new QueueData()); + responseBody.getBrokerDatas().add(new BrokerData()); + responseBody.getFilterServerTable().put("key", Collections.emptyList()); + Map topicQueueMappingByBroker = new HashMap<>(); + topicQueueMappingByBroker.put("key", new TopicQueueMappingInfo()); + responseBody.setTopicQueueMappingByBroker(topicQueueMappingByBroker); + setResponseBody(responseBody); + TopicRouteData actual = mqClientAPI.getTopicRouteInfoFromNameServer(defaultTopic, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getQueueDatas().size()); + assertEquals(1, actual.getBrokerDatas().size()); + assertEquals(1, actual.getFilterServerTable().size()); + assertEquals(1, actual.getTopicQueueMappingByBroker().size()); + } + + @Test + public void assertGetTopicListFromNameServer() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.setBrokerAddr(defaultBrokerAddr); + responseBody.getTopicList().add(defaultTopic); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getTopicListFromNameServer(defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicList().size()); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + } + + @Test + public void assertWipeWritePermOfBroker() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + WipeWritePermOfBrokerResponseHeader responseHeader = mock(WipeWritePermOfBrokerResponseHeader.class); + when(responseHeader.getWipeTopicCount()).thenReturn(1); + setResponseHeader(responseHeader); + assertEquals(1, mqClientAPI.wipeWritePermOfBroker(defaultNsAddr, brokerName, defaultTimeout)); + } + + @Test + public void testDeleteTopicInBroker() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.deleteTopicInBroker(defaultBrokerAddr, defaultTopic, defaultTimeout); + } + + @Test + public void testDeleteTopicInNameServer() throws RemotingException, InterruptedException, MQClientException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.deleteTopicInNameServer(defaultNsAddr, defaultTopic, defaultTimeout); + mqClientAPI.deleteTopicInNameServer(defaultNsAddr, clusterName, defaultTopic, defaultTimeout); + } + + @Test + public void testDeleteSubscriptionGroup() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.deleteSubscriptionGroup(defaultBrokerAddr, "", true, defaultTimeout); + } + + @Test + public void assertGetKVConfigValue() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + GetKVConfigResponseHeader responseHeader = mock(GetKVConfigResponseHeader.class); + when(responseHeader.getValue()).thenReturn("value"); + setResponseHeader(responseHeader); + assertEquals("value", mqClientAPI.getKVConfigValue("", "", defaultTimeout)); + } + + @Test + public void testPutKVConfigValue() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.putKVConfigValue("", "", "", defaultTimeout); + } + + @Test + public void testDeleteKVConfigValue() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.deleteKVConfigValue("", "", defaultTimeout); + } + + @Test + public void assertGetKVListByNamespace() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + KVTable responseBody = new KVTable(); + responseBody.getTable().put("key", "value"); + setResponseBody(responseBody); + KVTable actual = mqClientAPI.getKVListByNamespace("", defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTable().size()); + } + + @Test + public void assertInvokeBrokerToResetOffset() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + ResetOffsetBody responseBody = new ResetOffsetBody(); + responseBody.getOffsetTable().put(new MessageQueue(), 1L); + setResponseBody(responseBody); + Map actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, "", System.currentTimeMillis(), false, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + actual = mqClientAPI.invokeBrokerToResetOffset(defaultBrokerAddr, defaultTopic, "", System.currentTimeMillis(), 1, 1L, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + } + + @Test + public void assertInvokeBrokerToGetConsumerStatus() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + GetConsumerStatusBody responseBody = new GetConsumerStatusBody(); + responseBody.getConsumerTable().put("key", new HashMap<>()); + responseBody.getMessageQueueTable().put(new MessageQueue(), 1L); + setResponseBody(responseBody); + Map> actual = mqClientAPI.invokeBrokerToGetConsumerStatus(defaultBrokerAddr, defaultTopic, "", "", defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + } + + @Test + public void assertQueryTopicConsumeByWho() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + GroupList responseBody = new GroupList(); + responseBody.getGroupList().add(""); + setResponseBody(responseBody); + GroupList actual = mqClientAPI.queryTopicConsumeByWho(defaultBrokerAddr, defaultTopic, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getGroupList().size()); + } + + @Test + public void assertQueryTopicsByConsumer() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.getTopicList().add(defaultTopic); + responseBody.setBrokerAddr(defaultBrokerAddr); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.queryTopicsByConsumer(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicList().size()); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + } + + @Test + public void assertQuerySubscriptionByConsumer() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + QuerySubscriptionResponseBody responseBody = new QuerySubscriptionResponseBody(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTopic(defaultTopic); + responseBody.setSubscriptionData(subscriptionData); + setResponseBody(responseBody); + SubscriptionData actual = mqClientAPI.querySubscriptionByConsumer(defaultBrokerAddr, group, defaultTopic, defaultTimeout); + assertNotNull(actual); + assertEquals(defaultTopic, actual.getTopic()); + } + + @Test + public void assertQueryConsumeTimeSpan() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody(); + responseBody.getConsumeTimeSpanSet().add(new QueueTimeSpan()); + setResponseBody(responseBody); + List actual = mqClientAPI.queryConsumeTimeSpan(defaultBrokerAddr, defaultTopic, group, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + } + + @Test + public void assertGetTopicsByCluster() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.setBrokerAddr(defaultBrokerAddr); + responseBody.setTopicList(Collections.singleton(defaultTopic)); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getTopicsByCluster(clusterName, defaultTimeout); + assertNotNull(actual); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + assertEquals(1, actual.getTopicList().size()); + assertTrue(actual.getTopicList().contains(defaultTopic)); + } + + @Test + public void assertGetSystemTopicList() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.setBrokerAddr(defaultBrokerAddr); + responseBody.setTopicList(Collections.singleton(defaultTopic)); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getSystemTopicList(defaultTimeout); + assertNotNull(actual); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + assertEquals(1, actual.getTopicList().size()); + assertTrue(actual.getTopicList().contains(defaultTopic)); + } + + @Test + public void assertGetSystemTopicListFromBroker() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.setBrokerAddr(defaultBrokerAddr); + responseBody.setTopicList(Collections.singleton(defaultTopic)); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getSystemTopicListFromBroker(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + assertEquals(1, actual.getTopicList().size()); + assertTrue(actual.getTopicList().contains(defaultTopic)); + } + + @Test + public void assertCleanExpiredConsumeQueue() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + assertTrue(mqClientAPI.cleanExpiredConsumeQueue(defaultBrokerAddr, defaultTimeout)); + } + + @Test + public void assertDeleteExpiredCommitLog() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + assertTrue(mqClientAPI.deleteExpiredCommitLog(defaultBrokerAddr, defaultTimeout)); + } + + @Test + public void assertCleanUnusedTopicByAddr() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + assertTrue(mqClientAPI.cleanUnusedTopicByAddr(defaultBrokerAddr, defaultTimeout)); + } + + @Test + public void assertGetConsumerRunningInfo() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + ConsumerRunningInfo responseBody = new ConsumerRunningInfo(); + responseBody.setJstack("jstack"); + responseBody.getUserConsumerInfo().put("key", "value"); + setResponseBody(responseBody); + ConsumerRunningInfo actual = mqClientAPI.getConsumerRunningInfo(defaultBrokerAddr, group, clientId, false, defaultTimeout); + assertNotNull(actual); + assertEquals("jstack", actual.getJstack()); + assertEquals(1, actual.getUserConsumerInfo().size()); + } + + @Test + public void assertConsumeMessageDirectly() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult(); + responseBody.setAutoCommit(true); + responseBody.setRemark("remark"); + setResponseBody(responseBody); + ConsumeMessageDirectlyResult actual = mqClientAPI.consumeMessageDirectly(defaultBrokerAddr, group, clientId, topic, "", defaultTimeout); + assertNotNull(actual); + assertEquals("remark", actual.getRemark()); + assertTrue(actual.isAutoCommit()); + } + + @Test + public void assertQueryCorrectionOffset() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + QueryCorrectionOffsetBody responseBody = new QueryCorrectionOffsetBody(); + responseBody.getCorrectionOffsets().put(1, 1L); + setResponseBody(responseBody); + Map actual = mqClientAPI.queryCorrectionOffset(defaultBrokerAddr, topic, group, new HashSet<>(), defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + assertTrue(actual.containsKey(1)); + assertTrue(actual.containsValue(1L)); + } + + @Test + public void assertGetUnitTopicList() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.getTopicList().add(defaultTopic); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getUnitTopicList(false, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicList().size()); + } + + @Test + public void assertGetHasUnitSubTopicList() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.getTopicList().add(defaultTopic); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getHasUnitSubTopicList(false, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicList().size()); + } + + @Test + public void assertGetHasUnitSubUnUnitTopicList() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + TopicList responseBody = new TopicList(); + responseBody.getTopicList().add(defaultTopic); + setResponseBody(responseBody); + TopicList actual = mqClientAPI.getHasUnitSubUnUnitTopicList(false, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicList().size()); + } + + @Test + public void testCloneGroupOffset() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.cloneGroupOffset(defaultBrokerAddr, "", "", defaultTopic, false, defaultTimeout); + } + + @Test + public void assertViewBrokerStatsData() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + BrokerStatsData responseBody = new BrokerStatsData(); + responseBody.setStatsDay(new BrokerStatsItem()); + setResponseBody(responseBody); + BrokerStatsData actual = mqClientAPI.viewBrokerStatsData(defaultBrokerAddr, "", "", defaultTimeout); + assertNotNull(actual); + assertNotNull(actual.getStatsDay()); + } + + @Test + public void assertGetClusterList() { + Set actual = mqClientAPI.getClusterList(topic, defaultTimeout); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void assertFetchConsumeStatsInBroker() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + ConsumeStatsList responseBody = new ConsumeStatsList(); + responseBody.setBrokerAddr(defaultBrokerAddr); + responseBody.getConsumeStatsList().add(new HashMap<>()); + setResponseBody(responseBody); + ConsumeStatsList actual = mqClientAPI.fetchConsumeStatsInBroker(defaultBrokerAddr, false, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getConsumeStatsList().size()); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + } + + @Test + public void assertGetAllSubscriptionGroupForSubscriptionGroupWrapper() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + SubscriptionGroupWrapper responseBody = new SubscriptionGroupWrapper(); + responseBody.getSubscriptionGroupTable().put("key", new SubscriptionGroupConfig()); + setResponseBody(responseBody); + SubscriptionGroupWrapper actual = mqClientAPI.getAllSubscriptionGroup(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getSubscriptionGroupTable().size()); + assertNotNull(actual.getDataVersion()); + assertEquals(0, actual.getDataVersion().getStateVersion()); + } + + @Test + public void assertGetAllSubscriptionGroupForSubscriptionGroupConfig() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + SubscriptionGroupConfig responseBody = new SubscriptionGroupConfig(); + responseBody.setGroupName(group); + responseBody.setBrokerId(MixAll.MASTER_ID); + setResponseBody(responseBody); + SubscriptionGroupConfig actual = mqClientAPI.getSubscriptionGroupConfig(defaultBrokerAddr, group, defaultTimeout); + assertNotNull(actual); + assertEquals(group, actual.getGroupName()); + assertEquals(MixAll.MASTER_ID, actual.getBrokerId()); + } + + @Test + public void assertGetAllTopicConfig() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + TopicConfigSerializeWrapper responseBody = new TopicConfigSerializeWrapper(); + responseBody.getTopicConfigTable().put("key", new TopicConfig()); + setResponseBody(responseBody); + TopicConfigSerializeWrapper actual = mqClientAPI.getAllTopicConfig(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getTopicConfigTable().size()); + assertNotNull(actual.getDataVersion()); + assertEquals(0, actual.getDataVersion().getStateVersion()); + } + + @Test + public void testUpdateNameServerConfig() throws RemotingException, InterruptedException, MQClientException, UnsupportedEncodingException { + mockInvokeSync(); + mqClientAPI.updateNameServerConfig(createProperties(), Collections.singletonList(defaultNsAddr), defaultTimeout); + } + + @Test + public void assertGetNameServerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException { + mockInvokeSync(); + setResponseBody("{\"key\":\"value\"}"); + Map actual = mqClientAPI.getNameServerConfig(Collections.singletonList(defaultNsAddr), defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.size()); + assertTrue(actual.containsKey(defaultNsAddr)); + } + + @Test + public void assertQueryConsumeQueue() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + QueryConsumeQueueResponseBody responseBody = new QueryConsumeQueueResponseBody(); + responseBody.setQueueData(Collections.singletonList(new ConsumeQueueData())); + setResponseBody(responseBody); + QueryConsumeQueueResponseBody actual = mqClientAPI.queryConsumeQueue(defaultBrokerAddr, defaultTopic, 1, 1, 1, group, defaultTimeout); + assertNotNull(actual); + assertEquals(1, actual.getQueueData().size()); + } + + @Test + public void testCheckClientInBroker() throws RemotingException, InterruptedException, MQClientException { + mockInvokeSync(); + mqClientAPI.checkClientInBroker(defaultBrokerAddr, group, clientId, new SubscriptionData(), defaultTimeout); + } + + @Test + public void assertGetTopicConfig() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + TopicConfigAndQueueMapping responseBody = new TopicConfigAndQueueMapping(new TopicConfig(), new TopicQueueMappingDetail()); + setResponseBody(responseBody); + TopicConfigAndQueueMapping actual = mqClientAPI.getTopicConfig(defaultBrokerAddr, defaultTopic, defaultTimeout); + assertNotNull(actual); + assertNotNull(actual.getMappingDetail()); + } + + @Test + public void testCreateStaticTopic() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.createStaticTopic(defaultBrokerAddr, defaultTopic, new TopicConfig(), new TopicQueueMappingDetail(), false, defaultTimeout); + } + + @Test + public void assertUpdateAndGetGroupForbidden() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + GroupForbidden responseBody = new GroupForbidden(); + responseBody.setGroup(group); + responseBody.setTopic(defaultTopic); + setResponseBody(responseBody); + GroupForbidden actual = mqClientAPI.updateAndGetGroupForbidden(defaultBrokerAddr, new UpdateGroupForbiddenRequestHeader(), defaultTimeout); + assertNotNull(actual); + assertEquals(group, actual.getGroup()); + assertEquals(defaultTopic, actual.getTopic()); + } + + @Test + public void testResetMasterFlushOffset() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.resetMasterFlushOffset(defaultBrokerAddr, 1L); + } + + @Test + public void assertGetBrokerHAStatus() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + HARuntimeInfo responseBody = new HARuntimeInfo(); + responseBody.setMaster(true); + responseBody.setMasterCommitLogMaxOffset(1L); + setResponseBody(responseBody); + HARuntimeInfo actual = mqClientAPI.getBrokerHAStatus(defaultBrokerAddr, defaultTimeout); + assertNotNull(actual); + assertEquals(1L, actual.getMasterCommitLogMaxOffset()); + assertTrue(actual.isMaster()); + } + + @Test + public void assertGetControllerMetaData() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader(); + responseHeader.setGroup(group); + responseHeader.setIsLeader(true); + setResponseHeader(responseHeader); + GetMetaDataResponseHeader actual = mqClientAPI.getControllerMetaData(defaultBrokerAddr); + assertNotNull(actual); + assertEquals(group, actual.getGroup()); + assertTrue(actual.isLeader()); + } + + @Test + public void assertGetInSyncStateData() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + BrokerReplicasInfo responseBody = new BrokerReplicasInfo(); + BrokerReplicasInfo.ReplicasInfo replicasInfo = new BrokerReplicasInfo.ReplicasInfo(MixAll.MASTER_ID, defaultBrokerAddr, 1, 1, Collections.emptyList(), Collections.emptyList()); + responseBody.getReplicasInfoTable().put("key", replicasInfo); + GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader(); + responseHeader.setControllerLeaderAddress(defaultBrokerAddr); + setResponseHeader(responseHeader); + setResponseBody(responseBody); + BrokerReplicasInfo actual = mqClientAPI.getInSyncStateData(defaultBrokerAddr, Collections.singletonList(defaultBrokerAddr)); + assertNotNull(actual); + assertEquals(1L, actual.getReplicasInfoTable().size()); + } + + @Test + public void assertGetBrokerEpochCache() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + EpochEntryCache responseBody = new EpochEntryCache(clusterName, brokerName, MixAll.MASTER_ID, Collections.emptyList(), 1); + setResponseBody(responseBody); + EpochEntryCache actual = mqClientAPI.getBrokerEpochCache(defaultBrokerAddr); + assertNotNull(actual); + assertEquals(1L, actual.getMaxOffset()); + assertEquals(MixAll.MASTER_ID, actual.getBrokerId()); + assertEquals(brokerName, actual.getBrokerName()); + assertEquals(clusterName, actual.getClusterName()); + } + + @Test + public void assertGetControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException { + mockInvokeSync(); + setResponseBody("{\"key\":\"value\"}"); + Map actual = mqClientAPI.getControllerConfig(Collections.singletonList(defaultBrokerAddr), defaultTimeout); + assertNotNull(actual); + assertEquals(1L, actual.size()); + } + + @Test + public void testUpdateControllerConfig() throws RemotingException, InterruptedException, UnsupportedEncodingException, MQClientException { + mockInvokeSync(); + mqClientAPI.updateControllerConfig(createProperties(), Collections.singletonList(defaultBrokerAddr), defaultTimeout); + } + + @Test + public void assertElectMaster() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + BrokerMemberGroup responseBody = new BrokerMemberGroup(); + setResponseBody(responseBody); + GetMetaDataResponseHeader getMetaDataResponseHeader = new GetMetaDataResponseHeader(); + getMetaDataResponseHeader.setControllerLeaderAddress(defaultBrokerAddr); + when(response.decodeCommandCustomHeader(GetMetaDataResponseHeader.class)).thenReturn(getMetaDataResponseHeader); + ElectMasterResponseHeader responseHeader = new ElectMasterResponseHeader(); + when(response.decodeCommandCustomHeader(ElectMasterResponseHeader.class)).thenReturn(responseHeader); + Pair actual = mqClientAPI.electMaster(defaultBrokerAddr, clusterName, brokerName, MixAll.MASTER_ID); + assertNotNull(actual); + assertEquals(responseHeader, actual.getObject1()); + assertEquals(responseBody, actual.getObject2()); + } + + @Test + public void testCleanControllerBrokerData() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + GetMetaDataResponseHeader responseHeader = new GetMetaDataResponseHeader(); + responseHeader.setControllerLeaderAddress(defaultBrokerAddr); + setResponseHeader(responseHeader); + mqClientAPI.cleanControllerBrokerData(defaultBrokerAddr, clusterName, brokerName, "", false); + } + + @Test + public void testCreateUser() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.createUser(defaultBrokerAddr, new UserInfo(), defaultTimeout); + } + + @Test + public void testUpdateUser() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.updateUser(defaultBrokerAddr, new UserInfo(), defaultTimeout); + } + + @Test + public void testDeleteUser() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.deleteUser(defaultBrokerAddr, "", defaultTimeout); + } + + @Test + public void assertGetUser() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + setResponseBody(createUserInfo()); + UserInfo actual = mqClientAPI.getUser(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals("username", actual.getUsername()); + assertEquals("password", actual.getPassword()); + assertEquals("userStatus", actual.getUserStatus()); + assertEquals("userType", actual.getUserType()); + } + + @Test + public void assertListUser() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + setResponseBody(Collections.singletonList(createUserInfo())); + List actual = mqClientAPI.listUser(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals("username", actual.get(0).getUsername()); + assertEquals("password", actual.get(0).getPassword()); + assertEquals("userStatus", actual.get(0).getUserStatus()); + assertEquals("userType", actual.get(0).getUserType()); + } + + @Test + public void testCreateAcl() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.createAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout); + } + + @Test + public void testUpdateAcl() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.updateAcl(defaultBrokerAddr, new AclInfo(), defaultTimeout); + } + + @Test + public void testDeleteAcl() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + mqClientAPI.deleteAcl(defaultBrokerAddr, "", "", defaultTimeout); + } + + @Test + public void assertGetAcl() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + setResponseBody(createAclInfo()); + AclInfo actual = mqClientAPI.getAcl(defaultBrokerAddr, "", defaultTimeout); + assertNotNull(actual); + assertEquals("subject", actual.getSubject()); + assertEquals(1, actual.getPolicies().size()); + } + + @Test + public void assertListAcl() throws RemotingException, InterruptedException, MQBrokerException { + mockInvokeSync(); + setResponseBody(Collections.singletonList(createAclInfo())); + List actual = mqClientAPI.listAcl(defaultBrokerAddr, "", "", defaultTimeout); + assertNotNull(actual); + assertEquals("subject", actual.get(0).getSubject()); + assertEquals(1, actual.get(0).getPolicies().size()); + } + + private Properties createProperties() { + Properties result = new Properties(); + result.put("key", "value"); + return result; + } + + private AclInfo createAclInfo() { + return AclInfo.of("subject", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), ""); + } + + private UserInfo createUserInfo() { + UserInfo result = new UserInfo(); + result.setUsername("username"); + result.setPassword("password"); + result.setUserStatus("userStatus"); + result.setUserType("userType"); + return result; + } + + private void setResponseHeader(CommandCustomHeader responseHeader) throws RemotingCommandException { + when(response.decodeCommandCustomHeader(any())).thenReturn(responseHeader); + } + + private void setResponseBody(Object responseBody) { + when(response.getBody()).thenReturn(RemotingSerializable.encode(responseBody)); + } + + private void mockInvokeSync() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + when(response.getCode()).thenReturn(ResponseCode.SUCCESS); + when(response.getVersion()).thenReturn(1); + when(remotingClient.invokeSync(any(), any(), anyLong())).thenReturn(response); + when(remotingClient.getNameServerAddressList()).thenReturn(Collections.singletonList(defaultNsAddr)); + } + + private void setTopAddressing() throws NoSuchFieldException, IllegalAccessException { + TopAddressing topAddressing = mock(TopAddressing.class); + setField(mqClientAPI, "topAddressing", topAddressing); + when(topAddressing.fetchNSAddr()).thenReturn(defaultNsAddr); + } + + private void setField(final Object target, final String fieldName, final Object newValue) throws NoSuchFieldException, IllegalAccessException { + Class clazz = target.getClass(); + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, newValue); + } } From 3aa5d1936f1162afefccdf03fcc55bf0f7ee642c Mon Sep 17 00:00:00 2001 From: rongtong Date: Tue, 2 Jul 2024 17:08:54 +0800 Subject: [PATCH 046/265] Adjust the default value of ackMessageThreadPoolNums to 16 to prevent performance bottlenecks during high traffic. (#8337) --- .../src/main/java/org/apache/rocketmq/common/BrokerConfig.java | 2 +- store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 378301bedd2..3aac59e0a1b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -70,7 +70,7 @@ public class BrokerConfig extends BrokerIdentity { private int putMessageFutureThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4); private int pullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; private int litePullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; - private int ackMessageThreadPoolNums = 3; + private int ackMessageThreadPoolNums = 16; private int processReplyMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; private int queryMessageThreadPoolNums = 8 + PROCESSOR_NUMBER; diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index 453c9d1dc72..569cc3cfaa6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -816,7 +816,7 @@ private boolean putMessagePositionInfo(final long offset, final int size, final long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset(); if (expectLogicOffset < currentLogicOffset) { - log.warn("Build consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}", + log.warn("Build consume queue repeatedly, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}", expectLogicOffset, currentLogicOffset, this.topic, this.queueId, expectLogicOffset - currentLogicOffset); return true; } From 933ffc0b968d39329e8379b1ddc2fe3575550fda Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan Date: Wed, 3 Jul 2024 13:47:00 +0800 Subject: [PATCH 047/265] [ISSUE #8352] Fix CLIENT_REGISTER in registerConsumer (#8353) --- .../org/apache/rocketmq/broker/client/ConsumerManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java index 42e71e7e997..9f838b51544 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java @@ -178,8 +178,6 @@ public boolean registerConsumer(final String group, final ClientChannelInfo clie long start = System.currentTimeMillis(); ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); if (null == consumerGroupInfo) { - callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo, - subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet())); ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere); ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp); consumerGroupInfo = prev != null ? prev : tmp; @@ -188,6 +186,10 @@ public boolean registerConsumer(final String group, final ClientChannelInfo clie boolean r1 = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere); + if (r1) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo, + subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet())); + } boolean r2 = false; if (updateSubscription) { r2 = consumerGroupInfo.updateSubscription(subList); From 92c922378aa7c92e4239f0b46be8ea97ed257c2e Mon Sep 17 00:00:00 2001 From: weihubeats Date: Thu, 4 Jul 2024 08:53:07 +0800 Subject: [PATCH 048/265] [ISSUE #8358] Client does not send heartbeats to all Nameserve in clustered mode, resulting in frequent disconnections (#8359) * Adding null does not update * rolling back * remove client scanAvailableNameSrv --- .../client/impl/factory/MQClientInstance.java | 1 + .../remoting/netty/NettyClientConfig.java | 10 +++++++ .../remoting/netty/NettyRemotingClient.java | 30 +++++++++++-------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index b4ebf692736..c9fd3c83e04 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -152,6 +152,7 @@ public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String cli this.nettyClientConfig.setClientCallbackExecutorThreads(clientConfig.getClientCallbackExecutorThreads()); this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); + this.nettyClientConfig.setScanAvailableNameSrv(false); ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); ChannelEventListener channelEventListener; if (clientConfig.isEnableHeartbeatChannelEventListener()) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java index c28288786a3..7b7263e27a3 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java @@ -31,6 +31,8 @@ public class NettyClientConfig { private int connectTimeoutMillis = NettySystemConfig.connectTimeoutMillis; private long channelNotActiveInterval = 1000 * 60; + private boolean isScanAvailableNameSrv = true; + /** * IdleStateEvent will be triggered when neither read nor write was performed for * the specified period of this time. Specify {@code 0} to disable @@ -218,4 +220,12 @@ public String getSocksProxyConfig() { public void setSocksProxyConfig(String socksProxyConfig) { this.socksProxyConfig = socksProxyConfig; } + + public boolean isScanAvailableNameSrv() { + return isScanAvailableNameSrv; + } + + public void setScanAvailableNameSrv(boolean scanAvailableNameSrv) { + this.isScanAvailableNameSrv = scanAvailableNameSrv; + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 1bc5e57db52..1d595f32b9a 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -251,20 +251,24 @@ public void run(Timeout timeout) { }; this.timer.newTimeout(timerTaskScanResponseTable, 1000 * 3, TimeUnit.MILLISECONDS); - int connectTimeoutMillis = this.nettyClientConfig.getConnectTimeoutMillis(); - TimerTask timerTaskScanAvailableNameSrv = new TimerTask() { - @Override - public void run(Timeout timeout) { - try { - NettyRemotingClient.this.scanAvailableNameSrv(); - } catch (Exception e) { - LOGGER.error("scanAvailableNameSrv exception", e); - } finally { - timer.newTimeout(this, connectTimeoutMillis, TimeUnit.MILLISECONDS); + if (nettyClientConfig.isScanAvailableNameSrv()) { + int connectTimeoutMillis = this.nettyClientConfig.getConnectTimeoutMillis(); + TimerTask timerTaskScanAvailableNameSrv = new TimerTask() { + @Override + public void run(Timeout timeout) { + try { + NettyRemotingClient.this.scanAvailableNameSrv(); + } catch (Exception e) { + LOGGER.error("scanAvailableNameSrv exception", e); + } finally { + timer.newTimeout(this, connectTimeoutMillis, TimeUnit.MILLISECONDS); + } } - } - }; - this.timer.newTimeout(timerTaskScanAvailableNameSrv, 0, TimeUnit.MILLISECONDS); + }; + this.timer.newTimeout(timerTaskScanAvailableNameSrv, 0, TimeUnit.MILLISECONDS); + } + + } private Map.Entry getProxy(String addr) { From 9173def4040f00ea5a4f1e913382b8ccd2b08d17 Mon Sep 17 00:00:00 2001 From: rongtong Date: Thu, 4 Jul 2024 16:39:53 +0800 Subject: [PATCH 049/265] [ISSUE #8348] Allow custom fast-failure queues to be added in BrokerFastFailure (#8347) --- .../rocketmq/broker/BrokerController.java | 2 + .../broker/latency/BrokerFastFailure.java | 45 ++++++------ .../broker/latency/BrokerFastFailureTest.java | 68 ++++++++++++++++++- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 76224db5cb5..145a9522306 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -2519,4 +2519,6 @@ public ColdDataCgCtrService getColdDataCgCtrService() { public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { this.coldDataCgCtrService = coldDataCgCtrService; } + + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java index 0135ac929a7..ce8fdd88579 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java @@ -16,11 +16,15 @@ */ package org.apache.rocketmq.broker.latency; +import java.util.List; +import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.AbstractBrokerRunnable; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; @@ -42,13 +46,26 @@ public class BrokerFastFailure { private volatile long jstackTime = System.currentTimeMillis(); + private final List, Supplier>> cleanExpiredRequestQueueList = new ArrayList<>(); + public BrokerFastFailure(final BrokerController brokerController) { this.brokerController = brokerController; + initCleanExpiredRequestQueueList(); this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true, brokerController == null ? null : brokerController.getBrokerConfig())); } + private void initCleanExpiredRequestQueueList() { + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getSendThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getPullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getLitePullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getHeartbeatThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getEndTransactionThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAckThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue())); + cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAdminBrokerThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue())); + } + public static RequestTask castRunnable(final Runnable runnable) { try { if (runnable instanceof FutureTaskExt) { @@ -98,26 +115,9 @@ private void cleanExpiredRequest() { } } - cleanExpiredRequestInQueue(this.brokerController.getSendThreadPoolQueue(), - this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(), - this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getLitePullThreadPoolQueue(), - this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(), - this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this - .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getAckThreadPoolQueue(), - brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue()); - - cleanExpiredRequestInQueue(this.brokerController.getAdminBrokerThreadPoolQueue(), - brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue()); + for (Pair, Supplier> pair : cleanExpiredRequestQueueList) { + cleanExpiredRequestInQueue(pair.getObject1(), pair.getObject2().get()); + } } void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, final long maxWaitTimeMillsInQueue) { @@ -154,6 +154,11 @@ void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, fin } } + public synchronized void addCleanExpiredRequestQueue(BlockingQueue cleanExpiredRequestQueue, + Supplier maxWaitTimeMillsInQueueSupplier) { + cleanExpiredRequestQueueList.add(new Pair<>(cleanExpiredRequestQueue, maxWaitTimeMillsInQueueSupplier)); + } + public void shutdown() { this.scheduledExecutorService.shutdown(); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java index 31b547cf1be..2216a1d50c1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java @@ -19,16 +19,46 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.remoting.netty.RequestTask; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageStore; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; public class BrokerFastFailureTest { + + private BrokerController brokerController; + + private final BrokerConfig brokerConfig = new BrokerConfig(); + + private MessageStore messageStore; + + @Before + public void setUp() { + brokerController = Mockito.mock(BrokerController.class); + messageStore = Mockito.mock(DefaultMessageStore.class); + BlockingQueue queue = new LinkedBlockingQueue<>(); + Mockito.when(brokerController.getSendThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getPullThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getLitePullThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getHeartbeatThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getEndTransactionThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getAdminBrokerThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getAckThreadPoolQueue()).thenReturn(queue); + Mockito.when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + Mockito.when(messageStore.isOSPageCacheBusy()).thenReturn(false); + Mockito.when(brokerController.getMessageStore()).thenReturn(messageStore); + } + @Test public void testCleanExpiredRequestInQueue() throws Exception { - BrokerFastFailure brokerFastFailure = new BrokerFastFailure(null); + BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController); BlockingQueue queue = new LinkedBlockingQueue<>(); brokerFastFailure.cleanExpiredRequestInQueue(queue, 1); @@ -63,4 +93,40 @@ public void run() { assertThat(((FutureTaskExt) queue.peek()).getRunnable()).isEqualTo(requestTask); } + @Test + public void testCleanExpiredCustomRequestInQueue() throws Exception { + BrokerFastFailure brokerFastFailure = new BrokerFastFailure(brokerController); + brokerFastFailure.start(); + brokerConfig.setWaitTimeMillsInAckQueue(10); + BlockingQueue customThreadPoolQueue = new LinkedBlockingQueue<>(); + brokerFastFailure.addCleanExpiredRequestQueue(customThreadPoolQueue, () -> brokerConfig.getWaitTimeMillsInAckQueue()); + + Runnable runnable = new Runnable() { + @Override + public void run() { + + } + }; + RequestTask requestTask = new RequestTask(runnable, null, null); + customThreadPoolQueue.add(new FutureTaskExt<>(requestTask, null)); + + Thread.sleep(2000); + + assertThat(customThreadPoolQueue.size()).isEqualTo(0); + assertThat(requestTask.isStopRun()).isEqualTo(true); + + brokerConfig.setWaitTimeMillsInAckQueue(10000); + + RequestTask requestTask2 = new RequestTask(runnable, null, null); + customThreadPoolQueue.add(new FutureTaskExt<>(requestTask2, null)); + + Thread.sleep(1000); + + assertThat(customThreadPoolQueue.size()).isEqualTo(1); + assertThat(((FutureTaskExt) customThreadPoolQueue.peek()).getRunnable()).isEqualTo(requestTask2); + + brokerFastFailure.shutdown(); + + } + } \ No newline at end of file From 77d6633e622200ed52feef6fb2adeb12fba8e2c8 Mon Sep 17 00:00:00 2001 From: vate <806019582@qq.com> Date: Thu, 4 Jul 2024 18:36:55 +0800 Subject: [PATCH 050/265] [ISSUE #8298] Optimize some code format or style (#8299) Co-authored-by: supervate --- .../apache/rocketmq/acl/common/AclUtils.java | 8 +-- .../acl/plain/PlainPermissionManager.java | 67 +++++++++---------- .../rocketmq/broker/out/BrokerOuterAPI.java | 29 ++++---- .../org/apache/rocketmq/common/MixAll.java | 22 +++--- .../common/config/AbstractRocksDBStorage.java | 13 ++-- .../service/route/ProxyTopicRouteData.java | 18 +++-- 6 files changed, 70 insertions(+), 87 deletions(-) diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java index 65f04f54339..937619beee4 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java @@ -40,7 +40,7 @@ public class AclUtils { public static byte[] combineRequestContent(RemotingCommand request, SortedMap fieldsMap) { try { - StringBuilder sb = new StringBuilder(""); + StringBuilder sb = new StringBuilder(); for (Map.Entry entry : fieldsMap.entrySet()) { if (!SessionCredentials.SIGNATURE.equals(entry.getKey())) { sb.append(entry.getValue()); @@ -71,12 +71,12 @@ public static void IPv6AddressCheck(String netAddress) { if (isAsterisk(netAddress) || isMinus(netAddress)) { int asterisk = netAddress.indexOf("*"); int minus = netAddress.indexOf("-"); -// '*' must be the end of netAddress if it exists + // '*' must be the end of netAddress if it exists if (asterisk > -1 && asterisk != netAddress.length() - 1) { throw new AclException(String.format("NetAddress examine scope Exception netAddress is %s", netAddress)); } -// format like "2::ac5:78:1-200:*" or "2::ac5:78:1-200" is legal + // format like "2::ac5:78:1-200:*" or "2::ac5:78:1-200" is legal if (minus > -1) { if (asterisk == -1) { if (minus <= netAddress.lastIndexOf(":")) { @@ -128,7 +128,7 @@ public static String[] getAddresses(String netAddress, String partialAddress) { } public static boolean isScope(String netAddress, int index) { -// IPv6 Address + // IPv6 Address if (isColon(netAddress)) { netAddress = expandIP(netAddress, 8); String[] strArray = StringUtils.split(netAddress, ":"); diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index 345aed06c5a..b075e5364ee 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -94,7 +94,7 @@ public List getAllAclFiles(String path) { List allAclFileFullPath = new ArrayList<>(); File file = new File(path); File[] files = file.listFiles(); - for (int i = 0; i < files.length; i++) { + for (int i = 0; files != null && i < files.length; i++) { String fileName = files[i].getAbsolutePath(); File f = new File(fileName); if (fileName.equals(fileHome + MixAll.ACL_CONF_TOOLS_FILE)) { @@ -126,10 +126,9 @@ public void load() { fileList.add(defaultAclFile); } - for (int i = 0; i < fileList.size(); i++) { - final String currentFile = MixAll.dealFilePath(fileList.get(i)); - PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(currentFile, - PlainAccessData.class); + for (String path : fileList) { + final String currentFile = MixAll.dealFilePath(path); + PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(currentFile, PlainAccessData.class); if (plainAclConfData == null) { log.warn("No data in file {}", currentFile); continue; @@ -139,12 +138,11 @@ public void load() { List globalWhiteRemoteAddressStrategyList = new ArrayList<>(); List globalWhiteRemoteAddressesList = plainAclConfData.getGlobalWhiteRemoteAddresses(); if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { - for (int j = 0; j < globalWhiteRemoteAddressesList.size(); j++) { - globalWhiteRemoteAddressStrategyList.add(remoteAddressStrategyFactory. - getRemoteAddressStrategy(globalWhiteRemoteAddressesList.get(j))); + for (String address : globalWhiteRemoteAddressesList) { + globalWhiteRemoteAddressStrategyList.add(remoteAddressStrategyFactory.getRemoteAddressStrategy(address)); } } - if (globalWhiteRemoteAddressStrategyList.size() > 0) { + if (!globalWhiteRemoteAddressStrategyList.isEmpty()) { globalWhiteRemoteAddressStrategyMap.put(currentFile, globalWhiteRemoteAddressStrategyList); globalWhiteRemoteAddressStrategy.addAll(globalWhiteRemoteAddressStrategyList); } @@ -163,7 +161,7 @@ public void load() { } } } - if (plainAccessResourceMap.size() > 0) { + if (!plainAccessResourceMap.isEmpty()) { aclPlainAccessResourceMap.put(currentFile, plainAccessResourceMap); } @@ -219,17 +217,16 @@ public void load(String aclFilePath) { log.info("Broker plain acl conf data is : {}", plainAclConfData.toString()); List globalWhiteRemoteAddressesList = plainAclConfData.getGlobalWhiteRemoteAddresses(); if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { - for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) { - globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory. - getRemoteAddressStrategy(globalWhiteRemoteAddressesList.get(i))); + for (String address : globalWhiteRemoteAddressesList) { + globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory.getRemoteAddressStrategy(address)); } } this.globalWhiteRemoteAddressStrategy.addAll(globalWhiteRemoteAddressStrategy); if (this.globalWhiteRemoteAddressStrategyMap.get(aclFilePath) != null) { List remoteAddressStrategyList = this.globalWhiteRemoteAddressStrategyMap.get(aclFilePath); - for (int i = 0; i < remoteAddressStrategyList.size(); i++) { - this.globalWhiteRemoteAddressStrategy.remove(remoteAddressStrategyList.get(i)); + for (RemoteAddressStrategy remoteAddressStrategy : remoteAddressStrategyList) { + this.globalWhiteRemoteAddressStrategy.remove(remoteAddressStrategy); } this.globalWhiteRemoteAddressStrategyMap.put(aclFilePath, globalWhiteRemoteAddressStrategy); } @@ -279,11 +276,9 @@ public PlainAccessData updateAclConfigFileVersion(String aclFileName, PlainAcces List dataVersions = updateAclConfigMap.getDataVersion(); DataVersion dataVersion = new DataVersion(); - if (dataVersions != null) { - if (dataVersions.size() > 0) { - dataVersion.setTimestamp(dataVersions.get(0).getTimestamp()); - dataVersion.setCounter(new AtomicLong(dataVersions.get(0).getCounter())); - } + if (dataVersions != null && !dataVersions.isEmpty()) { + dataVersion.setTimestamp(dataVersions.get(0).getTimestamp()); + dataVersion.setCounter(new AtomicLong(dataVersions.get(0).getCounter())); } dataVersion.nextVersion(); List versionElement = new ArrayList<>(); @@ -336,7 +331,7 @@ public boolean updateAccessConfig(PlainAccessConfig plainAccessConfig) { if (accountMap == null) { accountMap = new HashMap<>(1); accountMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig)); - } else if (accountMap.size() == 0) { + } else if (accountMap.isEmpty()) { accountMap.put(plainAccessConfig.getAccessKey(), buildPlainAccessResource(plainAccessConfig)); } else { for (Map.Entry entry : accountMap.entrySet()) { @@ -469,7 +464,7 @@ public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList) { } public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList, String fileName) { - if (fileName == null || fileName.equals("")) { + if (fileName == null || fileName.isEmpty()) { fileName = this.defaultAclFile; } @@ -511,10 +506,8 @@ public AclConfig getAllAclConfig() { List whiteAddrs = new ArrayList<>(); Set accessKeySets = new HashSet<>(); - for (int i = 0; i < fileList.size(); i++) { - String path = fileList.get(i); - PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(path, - PlainAccessData.class); + for (String path : fileList) { + PlainAccessData plainAclConfData = AclUtils.getYamlDataObject(path, PlainAccessData.class); if (plainAclConfData == null) { continue; } @@ -525,18 +518,18 @@ public AclConfig getAllAclConfig() { List plainAccessConfigs = plainAclConfData.getAccounts(); if (plainAccessConfigs != null && !plainAccessConfigs.isEmpty()) { - for (int j = 0; j < plainAccessConfigs.size(); j++) { - if (!accessKeySets.contains(plainAccessConfigs.get(j).getAccessKey())) { - accessKeySets.add(plainAccessConfigs.get(j).getAccessKey()); + for (PlainAccessConfig accessConfig : plainAccessConfigs) { + if (!accessKeySets.contains(accessConfig.getAccessKey())) { + accessKeySets.add(accessConfig.getAccessKey()); PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); - plainAccessConfig.setGroupPerms(plainAccessConfigs.get(j).getGroupPerms()); - plainAccessConfig.setDefaultTopicPerm(plainAccessConfigs.get(j).getDefaultTopicPerm()); - plainAccessConfig.setDefaultGroupPerm(plainAccessConfigs.get(j).getDefaultGroupPerm()); - plainAccessConfig.setAccessKey(plainAccessConfigs.get(j).getAccessKey()); - plainAccessConfig.setSecretKey(plainAccessConfigs.get(j).getSecretKey()); - plainAccessConfig.setAdmin(plainAccessConfigs.get(j).isAdmin()); - plainAccessConfig.setTopicPerms(plainAccessConfigs.get(j).getTopicPerms()); - plainAccessConfig.setWhiteRemoteAddress(plainAccessConfigs.get(j).getWhiteRemoteAddress()); + plainAccessConfig.setGroupPerms(accessConfig.getGroupPerms()); + plainAccessConfig.setDefaultTopicPerm(accessConfig.getDefaultTopicPerm()); + plainAccessConfig.setDefaultGroupPerm(accessConfig.getDefaultGroupPerm()); + plainAccessConfig.setAccessKey(accessConfig.getAccessKey()); + plainAccessConfig.setSecretKey(accessConfig.getSecretKey()); + plainAccessConfig.setAdmin(accessConfig.isAdmin()); + plainAccessConfig.setTopicPerms(accessConfig.getTopicPerms()); + plainAccessConfig.setWhiteRemoteAddress(accessConfig.getWhiteRemoteAddress()); configs.add(plainAccessConfig); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index d1cdb297fed..d5c80ce2ec3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -336,7 +336,7 @@ public void sendHeartbeatViaDataVersion( final DataVersion dataVersion, final boolean isInBrokerContainer) { List nameServerAddressList = this.remotingClient.getAvailableNameSrvList(); - if (nameServerAddressList != null && nameServerAddressList.size() > 0) { + if (nameServerAddressList != null && !nameServerAddressList.isEmpty()) { final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader(); requestHeader.setBrokerAddr(brokerAddr); requestHeader.setBrokerName(brokerName); @@ -405,7 +405,7 @@ public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr) assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class); + ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class); return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress()); } default: @@ -574,8 +574,7 @@ private RegisterBrokerResult registerBroker( assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - RegisterBrokerResponseHeader responseHeader = - (RegisterBrokerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class); + RegisterBrokerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class); RegisterBrokerResult result = new RegisterBrokerResult(); result.setMasterAddr(responseHeader.getMasterAddr()); result.setHaServerAddr(responseHeader.getHaServerAddr()); @@ -725,7 +724,7 @@ public void run0() { switch (response.getCode()) { case ResponseCode.SUCCESS: { QueryDataVersionResponseHeader queryDataVersionResponseHeader = - (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class); + response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class); changed = queryDataVersionResponseHeader.getChanged(); byte[] body = response.getBody(); if (body != null) { @@ -887,7 +886,7 @@ public long getMaxOffset(final String addr, final String topic, final int queueI assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); + GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); return responseHeader.getOffset(); } @@ -909,7 +908,7 @@ public long getMinOffset(final String addr, final String topic, final int queueI assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); + GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); return responseHeader.getOffset(); } @@ -1096,8 +1095,7 @@ private SendResult processSendResponse( break; } if (sendStatus != null) { - SendMessageResponseHeader responseHeader = - (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + SendMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(SendMessageResponseHeader.class); //If namespace not null , reset Topic without namespace. String topic = msg.getTopic(); @@ -1270,7 +1268,7 @@ public Pair> brokerElect(String controllerA // Only record success response. case CONTROLLER_MASTER_STILL_EXIST: case SUCCESS: - final ElectMasterResponseHeader responseHeader = (ElectMasterResponseHeader) response.decodeCommandCustomHeader(ElectMasterResponseHeader.class); + final ElectMasterResponseHeader responseHeader = response.decodeCommandCustomHeader(ElectMasterResponseHeader.class); final ElectMasterResponseBody responseBody = RemotingSerializable.decode(response.getBody(), ElectMasterResponseBody.class); return new Pair<>(responseHeader, responseBody.getSyncStateSet()); } @@ -1285,7 +1283,7 @@ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, f final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); assert response != null; if (response.getCode() == SUCCESS) { - return (GetNextBrokerIdResponseHeader) response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class); + return response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class); } throw new MQBrokerException(response.getCode(), response.getRemark()); } @@ -1297,7 +1295,7 @@ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); assert response != null; if (response.getCode() == SUCCESS) { - return (ApplyBrokerIdResponseHeader) response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class); + return response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class); } throw new MQBrokerException(response.getCode(), response.getRemark()); } @@ -1310,7 +1308,7 @@ public Pair> registerBrokerT final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); assert response != null; if (response.getCode() == SUCCESS) { - RegisterBrokerToControllerResponseHeader responseHeader = (RegisterBrokerToControllerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class); + RegisterBrokerToControllerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class); Set syncStateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class).getSyncStateSet(); return new Pair<>(responseHeader, syncStateSet); } @@ -1328,7 +1326,7 @@ public Pair getReplicaInfo(final Str assert response != null; switch (response.getCode()) { case SUCCESS: { - final GetReplicaInfoResponseHeader header = (GetReplicaInfoResponseHeader) response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class); + final GetReplicaInfoResponseHeader header = response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class); assert response.getBody() != null; final SyncStateSet stateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class); return new Pair<>(header, stateSet); @@ -1447,8 +1445,7 @@ private PullResultExt processPullResponse( throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } - PullMessageResponseHeader responseHeader = - (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); + PullMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(PullMessageResponseHeader.class); return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(), responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta()); diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 47b4aac34a4..efb115509ac 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -120,21 +120,21 @@ public class MixAll { private static final String OS = System.getProperty("os.name").toLowerCase(); public static boolean isWindows() { - return OS.indexOf("win") >= 0; + return OS.contains("win"); } public static boolean isMac() { - return OS.indexOf("mac") >= 0; + return OS.contains("mac"); } public static boolean isUnix() { - return OS.indexOf("nix") >= 0 - || OS.indexOf("nux") >= 0 - || OS.indexOf("aix") > 0; + return OS.contains("nix") + || OS.contains("nux") + || OS.contains("aix"); } public static boolean isSolaris() { - return OS.indexOf("sunos") >= 0; + return OS.contains("sunos"); } public static String getWSAddr() { @@ -205,7 +205,7 @@ public static void string2FileNotSafe(final String str, final String fileName) t if (fileParent != null) { fileParent.mkdirs(); } - IOTinyUtils.writeStringToFile(file, str, "UTF-8"); + IOTinyUtils.writeStringToFile(file, str, DEFAULT_CHARSET); } public static String file2String(final String fileName) throws IOException { @@ -224,7 +224,7 @@ public static String file2String(final File file) throws IOException { } if (result) { - return new String(data, "UTF-8"); + return new String(data, DEFAULT_CHARSET); } } return null; @@ -364,9 +364,9 @@ public static void properties2Object(final Properties p, final Object object) { String property = p.getProperty(key); if (property != null) { Class[] pt = method.getParameterTypes(); - if (pt != null && pt.length > 0) { + if (pt.length > 0) { String cn = pt[0].getSimpleName(); - Object arg = null; + Object arg; if (cn.equals("int") || cn.equals("Integer")) { arg = Integer.parseInt(property); } else if (cn.equals("long") || cn.equals("Long")) { @@ -464,7 +464,7 @@ public static String getLocalhostByNetworkInterface() throws SocketException { return candidatesHost.get(0); } - // Fallback to loopback + // Fallback to loopback return localhost(); } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index 20319abba3d..ed3a12dc245 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -478,11 +478,7 @@ public void statRocksdb(Logger logger) { } Map map = Maps.newHashMap(); for (LiveFileMetaData metaData : liveFileMetaDataList) { - StringBuilder sb = map.get(metaData.level()); - if (sb == null) { - sb = new StringBuilder(256); - map.put(metaData.level(), sb); - } + StringBuilder sb = map.computeIfAbsent(metaData.level(), k -> new StringBuilder(256)); sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). append(metaData.fileName()).append(SPACE). append("s: ").append(metaData.size()).append(SPACE). @@ -491,9 +487,8 @@ public void statRocksdb(Logger logger) { append("d: ").append(metaData.numDeletions()).append(SPACE). append(metaData.beingCompacted()).append("\n"); } - for (Map.Entry entry : map.entrySet()) { - logger.info("level: {}\n{}", entry.getKey(), entry.getValue().toString()); - } + + map.forEach((key, value) -> logger.info("level: {}\n{}", key, value.toString())); String blockCacheMemUsage = this.db.getProperty("rocksdb.block-cache-usage"); String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem"); @@ -504,4 +499,4 @@ public void statRocksdb(Logger logger) { } catch (Exception ignored) { } } -} \ No newline at end of file +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java index 63651f6fe81..b5e65818ac5 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java @@ -40,12 +40,11 @@ public ProxyTopicRouteData(TopicRouteData topicRouteData) { ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData(); proxyBrokerData.setCluster(brokerData.getCluster()); proxyBrokerData.setBrokerName(brokerData.getBrokerName()); - for (Long brokerId : brokerData.getBrokerAddrs().keySet()) { - String brokerAddr = brokerData.getBrokerAddrs().get(brokerId); - HostAndPort hostAndPort = HostAndPort.fromString(brokerAddr); + brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> { + HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr); - proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, hostAndPort))); - } + proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, brokerHostAndPort))); + }); this.brokerDatas.add(proxyBrokerData); } } @@ -58,13 +57,12 @@ public ProxyTopicRouteData(TopicRouteData topicRouteData, int port) { ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData(); proxyBrokerData.setCluster(brokerData.getCluster()); proxyBrokerData.setBrokerName(brokerData.getBrokerName()); - for (Long brokerId : brokerData.getBrokerAddrs().keySet()) { - String brokerAddr = brokerData.getBrokerAddrs().get(brokerId); + brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> { HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr); - HostAndPort hostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), port); + HostAndPort proxyHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), port); - proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, hostAndPort))); - } + proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, proxyHostAndPort))); + }); this.brokerDatas.add(proxyBrokerData); } } From 77f6415eae1371c35b55bc52c1c24205f9c4a677 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 5 Jul 2024 15:33:30 +0800 Subject: [PATCH 051/265] [ISSUE #8360] Add more test coverage for MQAdminImpl (#8361) * [ISSUE #8360] Add more test coverage for MQAdminImpl * Update test * Update test --- .../rocketmq/client/impl/MQAdminImplTest.java | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java new file mode 100644 index 00000000000..3663df24d65 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl; + +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.QueryResult; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class MQAdminImplTest { + + @Mock + private MQClientInstance mQClientFactory; + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private MQAdminImpl mqAdminImpl; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultCluster = "defaultCluster"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final long defaultTimeout = 3000L; + + @Before + public void init() throws RemotingException, InterruptedException, MQClientException { + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createRouteData()); + ClientConfig clientConfig = mock(ClientConfig.class); + when(clientConfig.getNamespace()).thenReturn("namespace"); + when(mQClientFactory.getClientConfig()).thenReturn(clientConfig); + when(mQClientFactory.findBrokerAddressInPublish(any())).thenReturn(defaultBrokerAddr); + when(mQClientFactory.getAnExistTopicRouteData(any())).thenReturn(createRouteData()); + mqAdminImpl = new MQAdminImpl(mQClientFactory); + } + + @Test + public void assertTimeoutMillis() { + assertEquals(6000L, mqAdminImpl.getTimeoutMillis()); + mqAdminImpl.setTimeoutMillis(defaultTimeout); + assertEquals(defaultTimeout, mqAdminImpl.getTimeoutMillis()); + } + + @Test + public void testCreateTopic() throws MQClientException { + mqAdminImpl.createTopic("", defaultTopic, 6); + } + + @Test + public void assertFetchPublishMessageQueues() throws MQClientException { + List queueList = mqAdminImpl.fetchPublishMessageQueues(defaultTopic); + assertNotNull(queueList); + assertEquals(6, queueList.size()); + for (MessageQueue each : queueList) { + assertEquals(defaultTopic, each.getTopic()); + assertEquals(defaultBroker, each.getBrokerName()); + } + } + + @Test + public void assertFetchSubscribeMessageQueues() throws MQClientException { + Set queueList = mqAdminImpl.fetchSubscribeMessageQueues(defaultTopic); + assertNotNull(queueList); + assertEquals(6, queueList.size()); + for (MessageQueue each : queueList) { + assertEquals(defaultTopic, each.getTopic()); + assertEquals(defaultBroker, each.getBrokerName()); + } + } + + @Test + public void assertSearchOffset() throws MQClientException { + assertEquals(0, mqAdminImpl.searchOffset(new MessageQueue(), defaultTimeout)); + } + + @Test + public void assertMaxOffset() throws MQClientException { + assertEquals(0, mqAdminImpl.maxOffset(new MessageQueue())); + } + + @Test + public void assertMinOffset() throws MQClientException { + assertEquals(0, mqAdminImpl.minOffset(new MessageQueue())); + } + + @Test + public void assertEarliestMsgStoreTime() throws MQClientException { + assertEquals(0, mqAdminImpl.earliestMsgStoreTime(new MessageQueue())); + } + + @Test(expected = MQClientException.class) + public void assertViewMessage() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + MessageExt actual = mqAdminImpl.viewMessage(defaultTopic, "1"); + assertNotNull(actual); + } + + @Test + public void assertQueryMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); + QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader(); + responseHeader.setIndexLastUpdatePhyoffset(1L); + responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis()); + RemotingCommand response = mock(RemotingCommand.class); + when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader); + when(response.getBody()).thenReturn(getMessageResult()); + when(response.getCode()).thenReturn(ResponseCode.SUCCESS); + callback.operationSucceed(response); + return null; + }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any()); + QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L, false); + assertNotNull(actual); + assertEquals(1, actual.getMessageList().size()); + assertEquals(defaultTopic, actual.getMessageList().get(0).getTopic()); + } + + @Test + public void assertQueryMessageByUniqKey() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); + QueryMessageResponseHeader responseHeader = new QueryMessageResponseHeader(); + responseHeader.setIndexLastUpdatePhyoffset(1L); + responseHeader.setIndexLastUpdateTimestamp(System.currentTimeMillis()); + RemotingCommand response = mock(RemotingCommand.class); + when(response.decodeCommandCustomHeader(QueryMessageResponseHeader.class)).thenReturn(responseHeader); + when(response.getBody()).thenReturn(getMessageResult()); + when(response.getCode()).thenReturn(ResponseCode.SUCCESS); + callback.operationSucceed(response); + return null; + }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any()); + String msgId = buildMsgId(); + MessageExt actual = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId); + assertNotNull(actual); + assertEquals(msgId, actual.getMsgId()); + assertEquals(defaultTopic, actual.getTopic()); + actual = mqAdminImpl.queryMessageByUniqKey(defaultCluster, defaultTopic, msgId); + assertNotNull(actual); + assertEquals(msgId, actual.getMsgId()); + assertEquals(defaultTopic, actual.getTopic()); + QueryResult queryResult = mqAdminImpl.queryMessageByUniqKey(defaultTopic, msgId, 1, 0L, 1L); + assertNotNull(queryResult); + assertEquals(1, queryResult.getMessageList().size()); + assertEquals(defaultTopic, queryResult.getMessageList().get(0).getTopic()); + } + + private String buildMsgId() { + MessageExt msgExt = createMessageExt(); + int storeHostIPLength = (msgExt.getFlag() & MessageSysFlag.STOREHOSTADDRESS_V6_FLAG) == 0 ? 4 : 16; + int msgIDLength = storeHostIPLength + 4 + 8; + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); + return MessageDecoder.createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset()); + } + + private TopicRouteData createRouteData() { + TopicRouteData result = new TopicRouteData(); + result.setBrokerDatas(createBrokerData()); + result.setQueueDatas(createQueueData()); + return result; + } + + private List createBrokerData() { + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr); + return Collections.singletonList(new BrokerData(defaultCluster, defaultBroker, brokerAddrs)); + } + + private List createQueueData() { + QueueData queueData = new QueueData(); + queueData.setPerm(6); + queueData.setBrokerName(defaultBroker); + queueData.setReadQueueNums(6); + queueData.setWriteQueueNums(6); + return Collections.singletonList(queueData); + } + + private byte[] getMessageResult() throws Exception { + byte[] bytes = MessageDecoder.encode(createMessageExt(), false); + ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); + byteBuffer.put(bytes); + return byteBuffer.array(); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } +} From 2bc0014c4bf69b15bac9f33e03690e1bfce513c1 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Mon, 8 Jul 2024 21:09:20 +0800 Subject: [PATCH 052/265] [ISSUE #8377] Prepare to release Apache RocketMQ 5.3.0 (#8378) --- common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 5eb9dc06ac9..8ac75a72c98 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V5_2_0.ordinal(); + public static final int CURRENT_VERSION = Version.V5_3_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; From 7ee50ec32e10265c45e00df5971dd78b9598ff6e Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Mon, 8 Jul 2024 23:15:33 +0800 Subject: [PATCH 053/265] [maven-release-plugin] prepare release rocketmq-all-5.3.0 (#8379) --- acl/pom.xml | 2 +- auth/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- container/pom.xml | 2 +- controller/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- proxy/pom.xml | 2 +- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tieredstore/pom.xml | 2 +- tools/pom.xml | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index a52d6b66b48..3acd0aa7515 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 rocketmq-acl rocketmq-acl ${project.version} diff --git a/auth/pom.xml b/auth/pom.xml index 49f0fce7ab0..6c3cca9b061 100644 --- a/auth/pom.xml +++ b/auth/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 rocketmq-auth rocketmq-auth ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index ea9e35586dd..b5ac5ea5dcf 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index 13a92815586..669a4756d9d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 7be400394dd..1c275cd79e7 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/container/pom.xml b/container/pom.xml index c4863e207b7..421e9ffdb48 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/controller/pom.xml b/controller/pom.xml index df17bf97ebc..f323f8d5a48 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 jar diff --git a/distribution/pom.xml b/distribution/pom.xml index a475aa719de..c2876ad8faa 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 5b0ec76a547..069806c91f5 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/filter/pom.xml b/filter/pom.xml index 242428c6c80..db5400e3d11 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index eef4a32cadb..36517e5159c 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index bb9a27cfb39..81effc90932 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/pom.xml b/pom.xml index a72cf473f3a..01530651503 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -37,7 +37,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - HEAD + rocketmq-all-5.3.0 diff --git a/proxy/pom.xml b/proxy/pom.xml index 415c35233bb..04f87cdc235 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/remoting/pom.xml b/remoting/pom.xml index 342dcc7ea69..2bf23078bf5 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index e0174c63107..9514f961a7d 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 1f69a67d8c6..341c78fe120 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index df49d82d450..76e69162a62 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index 157e3397581..2f0ba087cd6 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 5fa31d02736..9a23f18039d 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.2.1-SNAPSHOT + 5.3.0 4.0.0 From 5dc0c6a8b57e6287aec88f5eca3b361d10b44969 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Tue, 9 Jul 2024 01:30:43 +0800 Subject: [PATCH 054/265] [maven-release-plugin] prepare for next development iteration (#8382) --- acl/pom.xml | 2 +- auth/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- container/pom.xml | 2 +- controller/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- proxy/pom.xml | 2 +- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tieredstore/pom.xml | 2 +- tools/pom.xml | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index 3acd0aa7515..c9d5085dcc1 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/auth/pom.xml b/auth/pom.xml index 6c3cca9b061..71b07c33750 100644 --- a/auth/pom.xml +++ b/auth/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT rocketmq-auth rocketmq-auth ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index b5ac5ea5dcf..7f74059a969 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index 669a4756d9d..5a6c92f97dd 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 1c275cd79e7..82994c9a197 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/container/pom.xml b/container/pom.xml index 421e9ffdb48..b9514defdb8 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/controller/pom.xml b/controller/pom.xml index f323f8d5a48..82b6fc7d969 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 jar diff --git a/distribution/pom.xml b/distribution/pom.xml index c2876ad8faa..60fc6170bbe 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 069806c91f5..7685a811690 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/filter/pom.xml b/filter/pom.xml index db5400e3d11..0acaa73f8ae 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 36517e5159c..d53540601e6 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 81effc90932..09ab5ed2586 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 01530651503..03a4ad18a13 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -37,7 +37,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-5.3.0 + HEAD diff --git a/proxy/pom.xml b/proxy/pom.xml index 04f87cdc235..41e6fa95f55 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/remoting/pom.xml b/remoting/pom.xml index 2bf23078bf5..566c983ea98 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 9514f961a7d..562a5ea2a33 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 341c78fe120..6de01626772 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index 76e69162a62..df380a0b604 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index 2f0ba087cd6..96f042da21b 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 9a23f18039d..ee459dfd95a 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.0 + 5.3.1-SNAPSHOT 4.0.0 From 67ddc1dae0548a5a2e413a2c7ef155b39a64f2b6 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 10 Jul 2024 15:23:48 +0800 Subject: [PATCH 055/265] [ISSUE #8375] Add more test coverage for MqClientAdminImpl (#8376) * Add more test coverage for MqClientAdminImpl --- .../impl/admin/MqClientAdminImplTest.java | 559 ++++++++++++++++++ 1 file changed, 559 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java new file mode 100644 index 00000000000..71682fb52c0 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/admin/MqClientAdminImplTest.java @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.admin; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueryConsumeTimeSpanBody; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.CreateTopicRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteTopicFromNamesrvRequestHeader; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class MqClientAdminImplTest { + + @Mock + private RemotingClient remotingClient; + + @Mock + private RemotingCommand response; + + private MqClientAdminImpl mqClientAdminImpl; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final long defaultTimeout = 3000L; + + @Before + public void init() throws RemotingException, InterruptedException, MQClientException { + mqClientAdminImpl = new MqClientAdminImpl(remotingClient); + when(remotingClient.invoke(any(String.class), any(RemotingCommand.class), any(Long.class))).thenReturn(CompletableFuture.completedFuture(response)); + } + + @Test + public void assertQueryMessageWithSuccess() throws Exception { + setResponseSuccess(getMessageResult()); + QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class); + when(requestHeader.getTopic()).thenReturn(defaultTopic); + when(requestHeader.getKey()).thenReturn("keys"); + CompletableFuture> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout); + List messageExtList = actual.get(); + assertNotNull(messageExtList); + assertEquals(1, messageExtList.size()); + } + + @Test + public void assertQueryMessageWithNotFound() throws Exception { + when(response.getCode()).thenReturn(ResponseCode.QUERY_NOT_FOUND); + QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout); + List messageExtList = actual.get(); + assertNotNull(messageExtList); + assertEquals(0, messageExtList.size()); + } + + @Test + public void assertQueryMessageWithError() { + setResponseError(); + QueryMessageRequestHeader requestHeader = mock(QueryMessageRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.queryMessage(defaultBrokerAddr, false, false, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertGetTopicStatsInfoWithSuccess() throws Exception { + TopicStatsTable responseBody = new TopicStatsTable(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout); + TopicStatsTable topicStatsTable = actual.get(); + assertNotNull(topicStatsTable); + assertEquals(0, topicStatsTable.getOffsetTable().size()); + } + + @Test + public void assertGetTopicStatsInfoWithError() { + setResponseError(); + GetTopicStatsInfoRequestHeader requestHeader = mock(GetTopicStatsInfoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getTopicStatsInfo(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertQueryConsumeTimeSpanWithSuccess() throws Exception { + QueryConsumeTimeSpanBody responseBody = new QueryConsumeTimeSpanBody(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout); + List queueTimeSpans = actual.get(); + assertNotNull(queueTimeSpans); + assertEquals(0, queueTimeSpans.size()); + } + + @Test + public void assertQueryConsumeTimeSpanWithError() { + setResponseError(); + QueryConsumeTimeSpanRequestHeader requestHeader = mock(QueryConsumeTimeSpanRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.queryConsumeTimeSpan(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertUpdateOrCreateTopicWithSuccess() throws Exception { + setResponseSuccess(null); + CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertUpdateOrCreateTopicWithError() { + setResponseError(); + CreateTopicRequestHeader requestHeader = mock(CreateTopicRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.updateOrCreateTopic(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertUpdateOrCreateSubscriptionGroupWithSuccess() throws Exception { + setResponseSuccess(null); + SubscriptionGroupConfig config = mock(SubscriptionGroupConfig.class); + CompletableFuture actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertUpdateOrCreateSubscriptionGroupWithError() { + setResponseError(); + SubscriptionGroupConfig config = mock(SubscriptionGroupConfig.class); + CompletableFuture actual = mqClientAdminImpl.updateOrCreateSubscriptionGroup(defaultBrokerAddr, config, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertDeleteTopicInBrokerWithSuccess() throws Exception { + setResponseSuccess(null); + DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertDeleteTopicInBrokerWithError() { + setResponseError(); + DeleteTopicRequestHeader requestHeader = mock(DeleteTopicRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteTopicInBroker(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertDeleteTopicInNameserverWithSuccess() throws Exception { + setResponseSuccess(null); + DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertDeleteTopicInNameserverWithError() { + setResponseError(); + DeleteTopicFromNamesrvRequestHeader requestHeader = mock(DeleteTopicFromNamesrvRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteTopicInNameserver(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertDeleteKvConfigWithSuccess() throws Exception { + setResponseSuccess(null); + DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertDeleteKvConfigWithError() { + setResponseError(); + DeleteKVConfigRequestHeader requestHeader = mock(DeleteKVConfigRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteKvConfig(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertDeleteSubscriptionGroupWithSuccess() throws Exception { + setResponseSuccess(null); + DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertDeleteSubscriptionGroupWithError() { + setResponseError(); + DeleteSubscriptionGroupRequestHeader requestHeader = mock(DeleteSubscriptionGroupRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.deleteSubscriptionGroup(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertInvokeBrokerToResetOffsetWithSuccess() throws Exception { + ResetOffsetBody responseBody = new ResetOffsetBody(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout); + assertEquals(0, actual.get().size()); + } + + @Test + public void assertInvokeBrokerToResetOffsetWithError() { + setResponseError(); + ResetOffsetRequestHeader requestHeader = mock(ResetOffsetRequestHeader.class); + CompletableFuture> actual = mqClientAdminImpl.invokeBrokerToResetOffset(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertViewMessageWithSuccess() throws Exception { + setResponseSuccess(getMessageResult()); + ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout); + MessageExt result = actual.get(); + assertNotNull(result); + assertEquals(defaultTopic, result.getTopic()); + } + + @Test + public void assertViewMessageWithError() { + setResponseError(); + ViewMessageRequestHeader requestHeader = mock(ViewMessageRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.viewMessage(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertGetBrokerClusterInfoWithSuccess() throws Exception { + ClusterInfo responseBody = new ClusterInfo(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + CompletableFuture actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout); + ClusterInfo result = actual.get(); + assertNotNull(result); + } + + @Test + public void assertGetBrokerClusterInfoWithError() { + setResponseError(); + CompletableFuture actual = mqClientAdminImpl.getBrokerClusterInfo(defaultBrokerAddr, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertGetConsumerConnectionListWithSuccess() throws Exception { + ConsumerConnection responseBody = new ConsumerConnection(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout); + ConsumerConnection result = actual.get(); + assertNotNull(result); + assertEquals(0, result.getConnectionSet().size()); + } + + @Test + public void assertGetConsumerConnectionListWithError() { + setResponseError(); + GetConsumerConnectionListRequestHeader requestHeader = mock(GetConsumerConnectionListRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumerConnectionList(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertQueryTopicsByConsumerWithSuccess() throws Exception { + TopicList responseBody = new TopicList(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout); + TopicList result = actual.get(); + assertNotNull(result); + assertEquals(0, result.getTopicList().size()); + } + + @Test + public void assertQueryTopicsByConsumerWithError() { + setResponseError(); + QueryTopicsByConsumerRequestHeader requestHeader = mock(QueryTopicsByConsumerRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.queryTopicsByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertQuerySubscriptionByConsumerWithSuccess() throws Exception { + SubscriptionData responseBody = new SubscriptionData(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout); + assertNull(actual.get()); + } + + @Test + public void assertQuerySubscriptionByConsumerWithError() { + setResponseError(); + QuerySubscriptionByConsumerRequestHeader requestHeader = mock(QuerySubscriptionByConsumerRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.querySubscriptionByConsumer(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertGetConsumeStatsWithSuccess() throws Exception { + ConsumeStats responseBody = new ConsumeStats(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout); + ConsumeStats result = actual.get(); + assertNotNull(result); + assertEquals(0, result.getOffsetTable().size()); + } + + @Test + public void assertGetConsumeStatsWithError() { + setResponseError(); + GetConsumeStatsRequestHeader requestHeader = mock(GetConsumeStatsRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumeStats(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertQueryTopicConsumeByWhoWithSuccess() throws Exception { + GroupList responseBody = new GroupList(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout); + GroupList result = actual.get(); + assertNotNull(result); + assertEquals(0, result.getGroupList().size()); + } + + @Test + public void assertQueryTopicConsumeByWhoWithError() { + setResponseError(); + QueryTopicConsumeByWhoRequestHeader requestHeader = mock(QueryTopicConsumeByWhoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.queryTopicConsumeByWho(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertGetConsumerRunningInfoWithSuccess() throws Exception { + ConsumerRunningInfo responseBody = new ConsumerRunningInfo(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout); + ConsumerRunningInfo result = actual.get(); + assertNotNull(result); + assertEquals(0, result.getProperties().size()); + } + + @Test + public void assertGetConsumerRunningInfoWithError() { + setResponseError(); + GetConsumerRunningInfoRequestHeader requestHeader = mock(GetConsumerRunningInfoRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.getConsumerRunningInfo(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + @Test + public void assertConsumeMessageDirectlyWithSuccess() throws Exception { + ConsumeMessageDirectlyResult responseBody = new ConsumeMessageDirectlyResult(); + setResponseSuccess(RemotingSerializable.encode(responseBody)); + ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout); + ConsumeMessageDirectlyResult result = actual.get(); + assertNotNull(result); + assertTrue(result.isAutoCommit()); + } + + @Test + public void assertConsumeMessageDirectlyWithError() { + setResponseError(); + ConsumeMessageDirectlyResultRequestHeader requestHeader = mock(ConsumeMessageDirectlyResultRequestHeader.class); + CompletableFuture actual = mqClientAdminImpl.consumeMessageDirectly(defaultBrokerAddr, requestHeader, defaultTimeout); + Throwable thrown = assertThrows(ExecutionException.class, actual::get); + assertTrue(thrown.getCause() instanceof MQClientException); + MQClientException mqException = (MQClientException) thrown.getCause(); + assertEquals(ResponseCode.SYSTEM_ERROR, mqException.getResponseCode()); + assertTrue(mqException.getMessage().contains("CODE: 1 DESC: null")); + } + + private byte[] getMessageResult() throws Exception { + byte[] bytes = MessageDecoder.encode(createMessageExt(), false); + ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); + byteBuffer.put(bytes); + return byteBuffer.array(); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName("defaultBroker"); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, "defaultGroup"); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } + + private void setResponseSuccess(byte[] body) { + when(response.getCode()).thenReturn(ResponseCode.SUCCESS); + when(response.getBody()).thenReturn(body); + } + + private void setResponseError() { + when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR); + } +} From 6c3781f17e22ec35ddb2113bab5cdc4967cb8260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=98=9F=E7=81=BF?= <37405937+qianye1001@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:47:58 +0800 Subject: [PATCH 056/265] [ISSUE #8365] add non-oneway updateConsumerOffset (#8368) --- .../client/impl/mqclient/MQClientAPIExt.java | 27 ++++++++++++++ .../impl/mqclient/MQClientAPIExtTest.java | 37 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java index b97e00c577f..0e2092b8a0f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java @@ -400,6 +400,33 @@ public CompletableFuture updateConsumerOffsetOneWay( return future; } + public CompletableFuture updateConsumerOffsetAsync( + String brokerAddr, + UpdateConsumerOffsetRequestHeader header, + long timeoutMillis + ) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, header); + CompletableFuture future = new CompletableFuture<>(); + invoke(brokerAddr, request, timeoutMillis).whenComplete((response, t) -> { + if (t != null) { + log.error("updateConsumerOffsetAsync failed, brokerAddr={}, requestHeader={}", brokerAddr, header, t); + future.completeExceptionally(t); + return; + } + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + future.complete(null); + } + case ResponseCode.SYSTEM_ERROR: + case ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST: + case ResponseCode.TOPIC_NOT_EXIST: { + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + } + }); + return future; + } + public CompletableFuture> getConsumerListByGroupAsync( String brokerAddr, GetConsumerListByGroupRequestHeader requestHeader, diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java index 752bc98eabd..6f692dff950 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java @@ -18,14 +18,19 @@ package org.apache.rocketmq.client.impl.mqclient; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,9 +39,12 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; @RunWith(MockitoJUnitRunner.class) public class MQClientAPIExtTest { @@ -71,4 +79,33 @@ public void sendMessageAsync() { CompletableFuture future = mqClientAPIExt.sendMessageAsync("127.0.0.1:10911", "test", msg, requestHeader, 10); assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingTimeoutException.class); } + + @Test + public void testUpdateConsumerOffsetAsync_Success() throws ExecutionException, InterruptedException { + CompletableFuture remotingFuture = new CompletableFuture<>(); + remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = mqClientAPIExt.updateConsumerOffsetAsync("brokerAddr", new UpdateConsumerOffsetRequestHeader(), 3000L); + + assertNull("Future should be completed without exception", future.get()); + } + + @Test + public void testUpdateConsumerOffsetAsync_Fail() throws InterruptedException { + + CompletableFuture remotingFuture = new CompletableFuture<>(); + remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "QueueId is null, topic is testTopic")); + doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletableFuture future = mqClientAPIExt.updateConsumerOffsetAsync("brokerAddr", new UpdateConsumerOffsetRequestHeader(), 3000L); + + try { + future.get(); + } catch (ExecutionException e) { + MQBrokerException customEx = (MQBrokerException) e.getCause(); + assertEquals(customEx.getResponseCode(), ResponseCode.SYSTEM_ERROR); + assertEquals(customEx.getErrorMessage(), "QueueId is null, topic is testTopic"); + } + } } \ No newline at end of file From 6f77f052b8574ce776b2d2ca1dc6eee0dbb19f55 Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 15 Jul 2024 20:18:28 +0800 Subject: [PATCH 057/265] [ISSUE #8384] Add more test coverage for ClientConfig (#8385) * [ISSUE #8384] Add more test coverage for ClientConfig * Update test * Update test * Update test --- .../rocketmq/client/ClientConfigTest.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java b/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java new file mode 100644 index 00000000000..5afe9cc011d --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/ClientConfigTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client; + +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class ClientConfigTest { + + private ClientConfig clientConfig; + + private final String resource = "resource"; + + @Before + public void init() { + clientConfig = createClientConfig(); + } + + @Test + public void testWithNamespace() { + Set resources = clientConfig.withNamespace(Collections.singleton(resource)); + assertTrue(resources.contains("lmq%resource")); + } + + @Test + public void testWithoutNamespace() { + String actual = clientConfig.withoutNamespace(resource); + assertEquals(resource, actual); + Set resources = clientConfig.withoutNamespace(Collections.singleton(resource)); + assertTrue(resources.contains(resource)); + } + + @Test + public void testQueuesWithNamespace() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setTopic("defaultTopic"); + Collection messageQueues = clientConfig.queuesWithNamespace(Collections.singleton(messageQueue)); + assertTrue(messageQueues.contains(messageQueue)); + assertEquals("lmq%defaultTopic", messageQueues.iterator().next().getTopic()); + } + + private ClientConfig createClientConfig() { + ClientConfig result = new ClientConfig(); + result.setUnitName("unitName"); + result.setClientIP("127.0.0.1"); + result.setClientCallbackExecutorThreads(1); + result.setPollNameServerInterval(1000 * 30); + result.setHeartbeatBrokerInterval(1000 * 30); + result.setPersistConsumerOffsetInterval(1000 * 5); + result.setPullTimeDelayMillsWhenException(1000); + result.setUnitMode(true); + result.setSocksProxyConfig("{}"); + result.setLanguage(LanguageCode.JAVA); + result.setDecodeReadBody(true); + result.setDecodeDecompressBody(true); + result.setAccessChannel(AccessChannel.LOCAL); + result.setMqClientApiTimeout(1000 * 3); + result.setEnableStreamRequestType(true); + result.setSendLatencyEnable(true); + result.setEnableHeartbeatChannelEventListener(true); + result.setDetectTimeout(200); + result.setDetectInterval(1000 * 2); + result.setUseHeartbeatV2(false); + result.buildMQClientId(); + result.setNamespace("lmq"); + return result; + } +} From 259efdb08691614ecdca201b7ecb20bfc15f6f15 Mon Sep 17 00:00:00 2001 From: Dongyuan Pan Date: Tue, 16 Jul 2024 17:24:14 +0800 Subject: [PATCH 058/265] [ISSUE #8350] Properties store error: crc32ReservedLength make undefine memory in properties when no enable AppendPropCRC (#8351) --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index c2150d7a321..1dd60523a58 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -1834,7 +1834,7 @@ class DefaultAppendMessageCallback implements AppendMessageCallback { private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; // Store the message content private final ByteBuffer msgStoreItemMemory; - private final int crc32ReservedLength = CommitLog.CRC32_RESERVED_LEN; + private final int crc32ReservedLength = enabledAppendPropCRC ? CommitLog.CRC32_RESERVED_LEN : 0; private final MessageStoreConfig messageStoreConfig; DefaultAppendMessageCallback(MessageStoreConfig messageStoreConfig) { From 1588d6576295963787a99ab8d8754adf458bf329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=98=9F=E7=81=BF?= <37405937+qianye1001@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:41:45 +0800 Subject: [PATCH 059/265] [ISSUE #8365] add remoting client non-oneway updateConsumerOffset function (#8391) * add non-oneway updateConsumerOffset --- .../proxy/processor/ConsumerProcessor.java | 29 +++++++++++++++---- .../processor/DefaultMessagingProcessor.java | 11 +++++-- .../proxy/processor/MessagingProcessor.java | 16 ++++++++-- .../message/ClusterMessageService.java | 15 ++++++++-- .../service/message/LocalMessageService.java | 6 ++++ .../proxy/service/message/MessageService.java | 7 +++++ 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java index 24fc0a2a28f..ace8af1b994 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java @@ -40,12 +40,12 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.common.ProxyException; import org.apache.rocketmq.proxy.common.ProxyExceptionCode; -import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.proxy.common.utils.ProxyUtils; import org.apache.rocketmq.proxy.service.ServiceManager; import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; @@ -96,7 +96,7 @@ public CompletableFuture popMessage( } return popMessage(ctx, messageQueue, consumerGroup, topic, maxMsgNums, invisibleTime, pollTime, initMode, subscriptionData, fifo, popMessageResultFilter, attemptId, timeoutMillis); - } catch (Throwable t) { + } catch (Throwable t) { future.completeExceptionally(t); } return future; @@ -287,7 +287,8 @@ public CompletableFuture> batchAckMessage( return FutureUtils.addExecutor(future, this.executor); } - protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { + protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, + String topic, List handleMessageList, long timeoutMillis) { return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) .thenApply(result -> { List results = new ArrayList<>(); @@ -393,6 +394,24 @@ public CompletableFuture updateConsumerOffset(ProxyContext ctx, MessageQue return FutureUtils.addExecutor(future, this.executor); } + public CompletableFuture updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long commitOffset, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(ctx, messageQueue); + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + requestHeader.setCommitOffset(commitOffset); + future = serviceManager.getMessageService().updateConsumerOffsetAsync(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + public CompletableFuture queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup, long timeoutMillis) { CompletableFuture future = new CompletableFuture<>(); @@ -501,9 +520,9 @@ public CompletableFuture getMinOffset(ProxyContext ctx, MessageQueue messa protected Set buildAddressableSet(ProxyContext ctx, Set mqSet) { Set addressableMessageQueueSet = new HashSet<>(mqSet.size()); - for (MessageQueue mq:mqSet) { + for (MessageQueue mq : mqSet) { try { - addressableMessageQueueSet.add(serviceManager.getTopicRouteService().buildAddressableMessageQueue(ctx, mq)) ; + addressableMessageQueueSet.add(serviceManager.getTopicRouteService().buildAddressableMessageQueue(ctx, mq)); } catch (Exception e) { log.error("build addressable message queue fail, messageQueue = {}", mq, e); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java index 48a732c284b..9c494d7a451 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java @@ -75,7 +75,7 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen protected ThreadPoolExecutor producerProcessorExecutor; protected ThreadPoolExecutor consumerProcessorExecutor; protected static final String ROCKETMQ_HOME = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, - System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); protected DefaultMessagingProcessor(ServiceManager serviceManager) { ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); @@ -167,7 +167,8 @@ public CompletableFuture forwardMessageToDeadLetterQueue(ProxyC } @Override - public CompletableFuture endTransaction(ProxyContext ctx, String topic, String transactionId, String messageId, String producerGroup, + public CompletableFuture endTransaction(ProxyContext ctx, String topic, String transactionId, + String messageId, String producerGroup, TransactionStatus transactionStatus, boolean fromTransactionCheck, long timeoutMillis) { return this.transactionProcessor.endTransaction(ctx, topic, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, timeoutMillis); @@ -225,6 +226,12 @@ public CompletableFuture updateConsumerOffset(ProxyContext ctx, MessageQue return this.consumerProcessor.updateConsumerOffset(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis); } + @Override + public CompletableFuture updateConsumerOffsetAsync(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long commitOffset, long timeoutMillis) { + return this.consumerProcessor.updateConsumerOffsetAsync(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis); + } + @Override public CompletableFuture queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java index 213d2beeeac..03d28262d73 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java @@ -33,10 +33,10 @@ import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.proxy.common.Address; import org.apache.rocketmq.proxy.common.MessageReceiptHandle; import org.apache.rocketmq.proxy.common.ProxyContext; -import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; import org.apache.rocketmq.proxy.service.metadata.MetadataService; import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; @@ -217,6 +217,14 @@ CompletableFuture updateConsumerOffset( long timeoutMillis ); + CompletableFuture updateConsumerOffsetAsync( + ProxyContext ctx, + MessageQueue messageQueue, + String consumerGroup, + long commitOffset, + long timeoutMillis + ); + CompletableFuture queryConsumerOffset( ProxyContext ctx, MessageQueue messageQueue, @@ -321,7 +329,9 @@ void addTransactionSubscription( MetadataService getMetadataService(); - void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle); + void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, + MessageReceiptHandle messageReceiptHandle); - MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, String receiptHandle); + MessageReceiptHandle removeReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, + String receiptHandle); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java index ba7d5ad8e28..f9eb94fcfce 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java @@ -29,10 +29,10 @@ import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.common.ProxyException; import org.apache.rocketmq.proxy.common.ProxyExceptionCode; -import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.proxy.service.route.TopicRouteService; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -139,7 +139,8 @@ public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle h } @Override - public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, + public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, + String consumerGroup, String topic, long timeoutMillis) { List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); return this.mqClientAPIFactory.getClient().batchAckMessageAsync( @@ -181,6 +182,16 @@ public CompletableFuture updateConsumerOffset(ProxyContext ctx, Addressabl ); } + @Override + public CompletableFuture updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().updateConsumerOffsetAsync( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + @Override public CompletableFuture> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, LockBatchRequestBody requestBody, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index aaa688fee64..6b2ba02f7c9 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -440,6 +440,12 @@ public CompletableFuture updateConsumerOffset(ProxyContext ctx, Addressabl throw new NotImplementedException("updateConsumerOffset is not implemented in LocalMessageService"); } + @Override + public CompletableFuture updateConsumerOffsetAsync(ProxyContext ctx, AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("updateConsumerOffsetAsync is not implemented in LocalMessageService"); + } + @Override public CompletableFuture> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, LockBatchRequestBody requestBody, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java index 58a835adb46..61accbc0412 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java @@ -120,6 +120,13 @@ CompletableFuture updateConsumerOffset( long timeoutMillis ); + CompletableFuture updateConsumerOffsetAsync( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, + long timeoutMillis + ); + CompletableFuture> lockBatchMQ( ProxyContext ctx, AddressableMessageQueue messageQueue, From 2d1e3143c7dc60aca805e6689c5364825e3020d4 Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+tanXiang003@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:08:20 +0800 Subject: [PATCH 060/265] [ISSUE #8372] Add more test coverage for AdminBrokerProcessor --- .../processor/AdminBrokerProcessor.java | 2 +- .../processor/AdminBrokerProcessorTest.java | 471 +++++++++++++++++- 2 files changed, 454 insertions(+), 19 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 1b29ff173cc..c5419a62df7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -1941,7 +1941,7 @@ private RemotingCommand getAllMessageRequestMode(ChannelHandlerContext ctx, Remo final RemotingCommand response = RemotingCommand.createResponseCommand(null); String content = this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager().encode(); - if (content != null && content.length() > 0) { + if (content != null && !content.isEmpty()) { try { response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); } catch (UnsupportedEncodingException e) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index e66703e5653..04324043fb8 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -20,21 +20,6 @@ import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.auth.authentication.enums.UserType; import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager; import org.apache.rocketmq.auth.authentication.model.Subject; @@ -45,8 +30,10 @@ import org.apache.rocketmq.auth.authorization.model.Environment; import org.apache.rocketmq.auth.authorization.model.Resource; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.broker.schedule.ScheduleMessageService; import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager; @@ -55,11 +42,13 @@ import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.constant.FIleReadaheadMode; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageAccessor; @@ -67,13 +56,20 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.body.AclInfo; +import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.remoting.protocol.body.QueryCorrectionOffsetBody; import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.UserInfo; import org.apache.rocketmq.remoting.protocol.header.CreateAclRequestHeader; @@ -82,8 +78,11 @@ import org.apache.rocketmq.remoting.protocol.header.DeleteAclRequestHeader; import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader; import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetRequestHeader; @@ -91,6 +90,13 @@ import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.NotifyMinBrokerIdChangeRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryCorrectionOffsetHeader; +import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateAclRequestHeader; @@ -98,12 +104,17 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.apache.rocketmq.store.stats.BrokerStats; +import org.apache.rocketmq.store.timer.TimerCheckpoint; +import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.TimerMetrics; +import org.apache.rocketmq.store.util.LibC; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -112,6 +123,26 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.LongAdder; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -170,11 +201,32 @@ public class AdminBrokerProcessorTest { @Mock private AuthorizationMetadataManager authorizationMetadataManager; + @Mock + private TimerMessageStore timerMessageStore; + + @Mock + private TimerMetrics timerMetrics; + + @Mock + private MessageStoreConfig messageStoreConfig; + + @Mock + private CommitLog commitLog; + + @Mock + private Broker2Client broker2Client; + + @Mock + private ClientChannelInfo clientChannelInfo; + @Before public void init() throws Exception { brokerController.setMessageStore(messageStore); brokerController.setAuthenticationMetadataManager(authenticationMetadataManager); brokerController.setAuthorizationMetadataManager(authorizationMetadataManager); + Field field = BrokerController.class.getDeclaredField("broker2Client"); + field.setAccessible(true); + field.set(brokerController, broker2Client); //doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor(); @@ -280,6 +332,31 @@ public void testUpdateAndCreateTopic() throws Exception { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } + @Test + public void testUpdateAndCreateTopicList() throws RemotingCommandException { + List systemTopicList = new ArrayList<>(systemTopicSet); + RemotingCommand request = buildCreateTopicListRequest(systemTopicList); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("The topic[" + systemTopicList.get(0) + "] is conflict with system topic."); + + List inValidTopicList = new ArrayList<>(); + inValidTopicList.add(""); + request = buildCreateTopicListRequest(inValidTopicList); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + List topicList = new ArrayList<>(); + topicList.add("TEST_CREATE_TOPIC"); + topicList.add("TEST_CREATE_TOPIC1"); + request = buildCreateTopicListRequest(topicList); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + //test no changes + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + @Test public void testDeleteTopicInRocksdb() throws Exception { if (notToBeExecuted()) { @@ -815,7 +892,6 @@ public void testCreateAcl() throws RemotingCommandException { request.setBody(JSON.toJSONBytes(aclInfo)); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - } @Test @@ -833,7 +909,6 @@ public void testUpdateAcl() throws RemotingCommandException { request.setBody(JSON.toJSONBytes(aclInfo)); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - } @Test @@ -893,6 +968,349 @@ public void testListAcl() throws RemotingCommandException { assertThat(aclInfoData.get(0).getPolicies().get(0).getEntries().get(0).getDecision()).isEqualTo("Allow"); } + @Test + public void testGetTimeCheckPoint() throws RemotingCommandException { + when(this.brokerController.getTimerCheckpoint()).thenReturn(null); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_CHECK_POINT, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("The checkpoint is null"); + + when(this.brokerController.getTimerCheckpoint()).thenReturn(new TimerCheckpoint()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + @Test + public void testGetTimeMetrics() throws RemotingCommandException, IOException { + when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(null); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TIMER_METRICS, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + when(this.brokerController.getMessageStore().getTimerMessageStore()).thenReturn(timerMessageStore); + when(this.timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics); + when(this.timerMetrics.encode()).thenReturn(new TimerMetrics.TimerMetricsSerializeWrapper().toJson(false)); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testUpdateColdDataFlowCtrGroupConfig() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_COLD_DATA_FLOW_CTR_CONFIG, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + request.setBody("consumerGroup1=1".getBytes()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + request.setBody("".getBytes()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testRemoveColdDataFlowCtrGroupConfig() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REMOVE_COLD_DATA_FLOW_CTR_CONFIG, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + request.setBody("consumerGroup1".getBytes()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetColdDataFlowCtrInfo() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_COLD_DATA_FLOW_CTR_INFO, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testSetCommitLogReadAheadMode() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_COMMITLOG_READ_MODE, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + HashMap extfields = new HashMap<>(); + extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_DONTNEED)); + request.setExtFields(extfields); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + extfields.clear(); + extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_NORMAL)); + request.setExtFields(extfields); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + this.brokerController.setMessageStore(defaultMessageStore); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + when(this.defaultMessageStore.getMessageStoreConfig()).thenReturn(messageStoreConfig); + when(this.defaultMessageStore.getCommitLog()).thenReturn(commitLog); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetUnknownCmdResponse() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(10000, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.REQUEST_CODE_NOT_SUPPORTED); + } + + @Test + public void testGetAllMessageRequestMode() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testResetOffset() throws RemotingCommandException { + ResetOffsetRequestHeader requestHeader = + createRequestHeader("topic","group",-1,false,-1,-1); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST); + + this.brokerController.getTopicConfigManager().getTopicConfigTable().put("topic", new TopicConfig("topic")); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + + this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().put("group", new SubscriptionGroupConfig()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + requestHeader.setQueueId(0); + request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader); + request.makeCustomHeaderToNet(); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + requestHeader.setOffset(2L); + request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader); + request.makeCustomHeaderToNet(); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + } + + @Test + public void testGetConsumerStatus() throws RemotingCommandException { + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + requestHeader.setGroup("group"); + requestHeader.setTopic("topic"); + requestHeader.setClientAddr(""); + RemotingCommand request = RemotingCommand + .createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader); + RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null); + responseCommand.setCode(ResponseCode.SUCCESS); + when(broker2Client.getConsumeStatus(anyString(),anyString(),anyString())).thenReturn(responseCommand); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testQueryTopicConsumeByWho() throws RemotingCommandException { + QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader(); + requestHeader.setTopic("topic"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader); + request.makeCustomHeaderToNet(); + HashSet groups = new HashSet<>(); + groups.add("group"); + when(brokerController.getConsumerManager()).thenReturn(consumerManager); + when(consumerManager.queryTopicConsumeByWho(anyString())).thenReturn(groups); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(RemotingSerializable.decode(response.getBody(), GroupList.class) + .getGroupList().contains("group")) + .isEqualTo(groups.contains("group")); + } + + @Test + public void testQueryTopicByConsumer() throws RemotingCommandException { + QueryTopicsByConsumerRequestHeader requestHeader = new QueryTopicsByConsumerRequestHeader(); + requestHeader.setGroup("group"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPICS_BY_CONSUMER, requestHeader); + request.makeCustomHeaderToNet(); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testQuerySubscriptionByConsumer() throws RemotingCommandException { + QuerySubscriptionByConsumerRequestHeader requestHeader = new QuerySubscriptionByConsumerRequestHeader(); + requestHeader.setGroup("group"); + requestHeader.setTopic("topic"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_SUBSCRIPTION_BY_CONSUMER, requestHeader); + request.makeCustomHeaderToNet(); + when(brokerController.getConsumerManager()).thenReturn(consumerManager); + when(consumerManager.findSubscriptionData(anyString(),anyString())).thenReturn(null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetSystemTopicListFromBroker() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testCleanExpiredConsumeQueue() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testDeleteExpiredCommitLog() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_EXPIRED_COMMITLOG, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testCleanUnusedTopic() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_UNUSED_TOPIC, null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetConsumerRunningInfo() throws RemotingCommandException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + when(brokerController.getConsumerManager()).thenReturn(consumerManager); + when(consumerManager.findChannel(anyString(),anyString())).thenReturn(null); + GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader(); + requestHeader.setClientId("client"); + requestHeader.setConsumerGroup("group"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + when(consumerManager.findChannel(anyString(),anyString())).thenReturn(clientChannelInfo); + when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V3_0_0_SNAPSHOT.ordinal()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + when(clientChannelInfo.getVersion()).thenReturn(MQVersion.Version.V5_2_3.ordinal()); + when(brokerController.getBroker2Client()).thenReturn(broker2Client); + when(clientChannelInfo.getChannel()).thenReturn(channel); + RemotingCommand responseCommand = RemotingCommand.createResponseCommand(null); + responseCommand.setCode(ResponseCode.SUCCESS); + when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenReturn(responseCommand); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + when(broker2Client.callClient(any(Channel.class),any(RemotingCommand.class))).thenThrow(new RemotingTimeoutException("timeout")); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.CONSUME_MSG_TIMEOUT); + } + + @Test + public void testQueryCorrectionOffset() throws RemotingCommandException { + Map correctionOffsetMap = new HashMap<>(); + correctionOffsetMap.put(0, 100L); + correctionOffsetMap.put(1, 200L); + Map compareOffsetMap = new HashMap<>(); + compareOffsetMap.put(0, 80L); + compareOffsetMap.put(1, 300L); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.queryMinOffsetInAllGroup(anyString(),anyString())).thenReturn(correctionOffsetMap); + when(consumerOffsetManager.queryOffset(anyString(),anyString())).thenReturn(compareOffsetMap); + QueryCorrectionOffsetHeader queryCorrectionOffsetHeader = new QueryCorrectionOffsetHeader(); + queryCorrectionOffsetHeader.setTopic("topic"); + queryCorrectionOffsetHeader.setCompareGroup("group"); + queryCorrectionOffsetHeader.setFilterGroups(""); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, queryCorrectionOffsetHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + QueryCorrectionOffsetBody body = RemotingSerializable.decode(response.getBody(), QueryCorrectionOffsetBody.class); + Map correctionOffsets = body.getCorrectionOffsets(); + assertThat(correctionOffsets.get(0)).isEqualTo(Long.MAX_VALUE); + assertThat(correctionOffsets.get(1)).isEqualTo(200L); + } + + @Test + public void testNotifyMinBrokerIdChange() throws RemotingCommandException { + NotifyMinBrokerIdChangeRequestHeader requestHeader = new NotifyMinBrokerIdChangeRequestHeader(); + requestHeader.setMinBrokerId(1L); + requestHeader.setMinBrokerAddr("127.0.0.1:10912"); + requestHeader.setOfflineBrokerAddr("127.0.0.1:10911"); + requestHeader.setHaBrokerAddr(""); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testUpdateBrokerHaInfo() throws RemotingCommandException { + ExchangeHAInfoResponseHeader requestHeader = new ExchangeHAInfoResponseHeader(); + requestHeader.setMasterAddress("127.0.0.1:10911"); + requestHeader.setMasterFlushOffset(0L); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + when(brokerController.getMessageStore()).thenReturn(messageStore); + requestHeader.setMasterHaAddress("127.0.0.1:10912"); + request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader); + request.makeCustomHeaderToNet(); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + when(messageStore.getMasterFlushedOffset()).thenReturn(0L); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetBrokerHaStatus() throws RemotingCommandException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_HA_STATUS,null); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + when(brokerController.getMessageStore()).thenReturn(messageStore); + when(messageStore.getHARuntimeInfo()).thenReturn(new HARuntimeInfo()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testResetMasterFlushOffset() throws RemotingCommandException { + ResetMasterFlushOffsetHeader requestHeader = new ResetMasterFlushOffsetHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RESET_MASTER_FLUSH_OFFSET,requestHeader); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + requestHeader.setMasterFlushOffset(0L); + request.makeCustomHeaderToNet(); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + private ResetOffsetRequestHeader createRequestHeader(String topic,String group,long timestamp,boolean force,long offset,int queueId) { + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setTimestamp(timestamp); + requestHeader.setForce(force); + requestHeader.setOffset(offset); + requestHeader.setQueueId(queueId); + return requestHeader; + } + private RemotingCommand buildCreateTopicRequest(String topic) { CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); requestHeader.setTopic(topic); @@ -900,12 +1318,29 @@ private RemotingCommand buildCreateTopicRequest(String topic) { requestHeader.setReadQueueNums(8); requestHeader.setWriteQueueNums(8); requestHeader.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); request.makeCustomHeaderToNet(); return request; } + private RemotingCommand buildCreateTopicListRequest(List topicList) { + List topicConfigList = new ArrayList<>(); + for (String topic:topicList) { + TopicConfig topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(8); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG); + topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); + topicConfig.setTopicSysFlag(0); + topicConfig.setOrder(false); + topicConfigList.add(topicConfig); + } + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, null); + CreateTopicListRequestBody createTopicListRequestBody = new CreateTopicListRequestBody(topicConfigList); + request.setBody(createTopicListRequestBody.encode()); + return request; + } + private RemotingCommand buildDeleteTopicRequest(String topic) { DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); requestHeader.setTopic(topic); From 852b635e515de223fcfce13c3c87b0bc2061ac49 Mon Sep 17 00:00:00 2001 From: yx9o Date: Thu, 18 Jul 2024 11:11:58 +0800 Subject: [PATCH 061/265] [ISSUE #8396] Fix typo in TraceConstants (#8398) * [ISSUE #8396] Fix typo in TraceConstants * Update * Update * Update * Update * Update * Update --- .../java/org/apache/rocketmq/client/trace/TraceConstants.java | 2 +- .../client/trace/hook/EndTransactionOpenTracingHookImpl.java | 2 +- .../client/trace/hook/SendMessageOpenTracingHookImpl.java | 2 +- .../client/trace/DefaultMQProducerWithOpenTracingTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java index 1ad4b610515..67f7ab3f8fb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceConstants.java @@ -32,7 +32,7 @@ public class TraceConstants { public static final String ROCKETMQ_SUCCESS = "rocketmq.success"; public static final String ROCKETMQ_TAGS = "rocketmq.tags"; public static final String ROCKETMQ_KEYS = "rocketmq.keys"; - public static final String ROCKETMQ_SOTRE_HOST = "rocketmq.store_host"; + public static final String ROCKETMQ_STORE_HOST = "rocketmq.store_host"; public static final String ROCKETMQ_BODY_LENGTH = "rocketmq.body_length"; public static final String ROCKETMQ_MSG_ID = "rocketmq.mgs_id"; public static final String ROCKETMQ_MSG_TYPE = "rocketmq.mgs_type"; diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java index 62d310f1961..44e4b69776e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/EndTransactionOpenTracingHookImpl.java @@ -60,7 +60,7 @@ public void endTransaction(EndTransactionContext context) { span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic()); span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags()); span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); - span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr()); + span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr()); span.setTag(TraceConstants.ROCKETMQ_MSG_ID, context.getMsgId()); span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, MessageType.Trans_msg_Commit.name()); span.setTag(TraceConstants.ROCKETMQ_TRANSACTION_ID, context.getTransactionId()); diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java index 60c18a22a77..3cb64933849 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java @@ -60,7 +60,7 @@ public void sendMessageBefore(SendMessageContext context) { span.setTag(Tags.MESSAGE_BUS_DESTINATION, msg.getTopic()); span.setTag(TraceConstants.ROCKETMQ_TAGS, msg.getTags()); span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); - span.setTag(TraceConstants.ROCKETMQ_SOTRE_HOST, context.getBrokerAddr()); + span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr()); span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, context.getMsgType().name()); span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getBody().length); context.setMqTraceContext(span); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java index 9ce9d6b4941..c0eb8568dc6 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java @@ -124,7 +124,7 @@ public void testSendMessageSync_WithTrace_Success() throws RemotingException, In assertThat(span.tags().get(TraceConstants.ROCKETMQ_BODY_LENGTH)).isEqualTo(3); assertThat(span.tags().get(TraceConstants.ROCKETMQ_REGION_ID)).isEqualTo("HZ"); assertThat(span.tags().get(TraceConstants.ROCKETMQ_MSG_TYPE)).isEqualTo(MessageType.Normal_Msg.name()); - assertThat(span.tags().get(TraceConstants.ROCKETMQ_SOTRE_HOST)).isEqualTo("127.0.0.1:10911"); + assertThat(span.tags().get(TraceConstants.ROCKETMQ_STORE_HOST)).isEqualTo("127.0.0.1:10911"); } @After From cf4234b288ff3ee64446e1d776052fcb0c87510a Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+tanXiang003@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:26:33 +0800 Subject: [PATCH 062/265] [ISSUE #8392] add tests for QueryMessageProcessor --- .../processor/QueryMessageProcessorTest.java | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java new file mode 100644 index 00000000000..0fd54df7d8a --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryMessageProcessorTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.QueryMessageResult; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class QueryMessageProcessorTest { + private QueryMessageProcessor queryMessageProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private MessageStore messageStore; + + @Mock + private ChannelHandlerContext handlerContext; + + @Mock + private Channel channel; + + @Mock + private ChannelFuture channelFuture; + + @Before + public void init() { + when(handlerContext.channel()).thenReturn(channel); + queryMessageProcessor = new QueryMessageProcessor(brokerController); + when(brokerController.getMessageStore()).thenReturn(messageStore); + when(channel.writeAndFlush(any())).thenReturn(channelFuture); + } + + @Test + public void testQueryMessage() throws RemotingCommandException { + QueryMessageResult result = new QueryMessageResult(); + result.setIndexLastUpdateTimestamp(100); + result.setIndexLastUpdatePhyoffset(0); + result.addMessage(new SelectMappedBufferResult(0, null, 0, null)); + + when(messageStore.queryMessage(anyString(),anyString(),anyInt(),anyLong(),anyLong())).thenReturn(result); + RemotingCommand request = createQueryMessageRequest("topic", "msgKey", 1, 100, 200,"false"); + request.makeCustomHeaderToNet(); + RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(response.getCode(), ResponseCode.QUERY_NOT_FOUND); + + result.addMessage(new SelectMappedBufferResult(0, null, 1, null)); + when(messageStore.queryMessage(anyString(),anyString(),anyInt(),anyLong(),anyLong())).thenReturn(result); + response = queryMessageProcessor.processRequest(handlerContext, request); + Assert.assertNull(response); + } + + @Test + public void testViewMessageById() throws RemotingCommandException { + ViewMessageRequestHeader viewMessageRequestHeader = new ViewMessageRequestHeader(); + viewMessageRequestHeader.setTopic("topic"); + viewMessageRequestHeader.setOffset(0L); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, viewMessageRequestHeader); + request.makeCustomHeaderToNet(); + request.setCode(RequestCode.VIEW_MESSAGE_BY_ID); + + when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(null); + RemotingCommand response = queryMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(response.getCode(), ResponseCode.SYSTEM_ERROR); + + when(messageStore.selectOneMessageByOffset(anyLong())).thenReturn(new SelectMappedBufferResult(0, null, 0, null)); + response = queryMessageProcessor.processRequest(handlerContext, request); + Assert.assertNull(response); + } + + private RemotingCommand createQueryMessageRequest(String topic, String key, int maxNum, long beginTimestamp, long endTimestamp,String flag) { + QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setKey(key); + requestHeader.setMaxNum(maxNum); + requestHeader.setBeginTimestamp(beginTimestamp); + requestHeader.setEndTimestamp(endTimestamp); + + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.UNIQUE_MSG_QUERY_FLAG, flag); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader); + request.setExtFields(extFields); + return request; + } +} From 9115d662d0495d3d6269e7825ec829bd6d38dfcf Mon Sep 17 00:00:00 2001 From: sheep_yan <313169664@qq.com> Date: Thu, 18 Jul 2024 13:53:48 +0800 Subject: [PATCH 063/265] [ISSUE #8366] Eliminate deadlocks during the client shutdown process. (#8367) * [ISSUE #8366] When determining if `ChannelWrapper` is the wrapper for a channel, no longer acquire a read lock. * [ISSUE #8366] Compare channels for equality using `isWrapperOf`. --- .../remoting/netty/NettyRemotingClient.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 1d595f32b9a..41976122b2f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -421,7 +421,7 @@ public void closeChannel(final String addr, final Channel channel) { if (null == prevCW) { LOGGER.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote); removeItemFromTable = false; - } else if (prevCW.getChannel() != channel) { + } else if (prevCW.isWrapperOf(channel)) { LOGGER.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.", addrRemote); removeItemFromTable = false; @@ -463,12 +463,10 @@ public void closeChannel(final Channel channel) { for (Map.Entry entry : channelTables.entrySet()) { String key = entry.getKey(); ChannelWrapper prev = entry.getValue(); - if (prev.getChannel() != null) { - if (prev.getChannel() == channel) { - prevCW = prev; - addrRemote = key; - break; - } + if (prev.isWrapperOf(channel)) { + prevCW = prev; + addrRemote = key; + break; } } @@ -1022,6 +1020,13 @@ public boolean isWritable() { return getChannel().isWritable(); } + public boolean isWrapperOf(Channel channel) { + if (this.channelFuture.channel() != null && this.channelFuture.channel() == channel) { + return true; + } + return false; + } + private Channel getChannel() { return getChannelFuture().channel(); } From 73d0c33f4f1ab5e2b57afa4afc6203000aebeabd Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+tanXiang003@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:21:09 +0800 Subject: [PATCH 064/265] add tests for ConsumerManageProcessor (#8401) --- .../ConsumerManageProcessorTest.java | 186 +++++++++++++++++- 1 file changed, 185 insertions(+), 1 deletion(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java index c94591d381d..6b3c2578af3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java @@ -16,19 +16,36 @@ */ package org.apache.rocketmq.broker.processor; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.broker.topic.TopicQueueMappingManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerListByGroupRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.remoting.rpc.RpcClient; +import org.apache.rocketmq.remoting.rpc.RpcException; +import org.apache.rocketmq.remoting.rpc.RpcResponse; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.Before; @@ -38,7 +55,17 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -50,12 +77,24 @@ public class ConsumerManageProcessorTest { private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); @Mock private MessageStore messageStore; + @Mock + private Channel channel; + @Mock + private ConsumerOffsetManager consumerOffsetManager; + @Mock + private BrokerOuterAPI brokerOuterAPI; + @Mock + private RpcClient rpcClient; + @Mock + private Future responseFuture; + @Mock + private TopicQueueMappingContext mappingContext; private String topic = "FooBar"; private String group = "FooBarGroup"; @Before - public void init() { + public void init() throws RpcException { brokerController.setMessageStore(messageStore); TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController); topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic)); @@ -64,6 +103,12 @@ public void init() { subscriptionGroupManager.getSubscriptionGroupTable().put(group, new SubscriptionGroupConfig()); when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager); consumerManageProcessor = new ConsumerManageProcessor(brokerController); + when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI); + when(brokerOuterAPI.getRpcClient()).thenReturn(rpcClient); + when(rpcClient.invoke(any(),anyLong())).thenReturn(responseFuture); + TopicQueueMappingDetail topicQueueMappingDetail = new TopicQueueMappingDetail(); + topicQueueMappingDetail.setBname("BrokerA"); + when(mappingContext.getMappingDetail()).thenReturn(topicQueueMappingDetail); } @Test @@ -82,6 +127,145 @@ public void testUpdateConsumerOffset_GroupNotExist() throws Exception { assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); } + @Test + public void testUpdateConsumerOffset() throws RemotingCommandException { + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(true); + RemotingCommand request = buildUpdateConsumerOffsetRequest(group, topic, 0, 0); + RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + when(consumerOffsetManager.hasOffsetReset(anyString(),anyString(),anyInt())).thenReturn(false); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetConsumerListByGroup() throws RemotingCommandException { + GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader(); + requestHeader.setConsumerGroup(group); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + brokerController.getConsumerManager().getConsumerTable().put(group,new ConsumerGroupInfo(group)); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo( + requestHeader.getConsumerGroup()); + consumerGroupInfo.getChannelInfoTable().put(channel,new ClientChannelInfo(channel)); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testQueryConsumerOffset() throws RemotingCommandException, ExecutionException, InterruptedException { + RemotingCommand request = buildQueryConsumerOffsetRequest(group, topic, 0, true); + RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND); + + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(0L); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + when(consumerOffsetManager.queryOffset(anyString(),anyString(),anyInt())).thenReturn(-1L); + when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(-1L); + when(messageStore.checkInMemByConsumeOffset(anyString(),anyInt(),anyLong(),anyInt())).thenReturn(true); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + TopicQueueMappingManager topicQueueMappingManager = mock(TopicQueueMappingManager.class); + when(brokerController.getTopicQueueMappingManager()).thenReturn(topicQueueMappingManager); + when(topicQueueMappingManager.buildTopicQueueMappingContext(any(QueryConsumerOffsetRequestHeader.class))).thenReturn(mappingContext); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE); + + List items = new ArrayList<>(); + LogicQueueMappingItem item1 = createLogicQueueMappingItem("BrokerC", 0, 0L, 0L); + items.add(item1); + when(mappingContext.getMappingItemList()).thenReturn(items); + when(mappingContext.getLeaderItem()).thenReturn(item1); + when(mappingContext.getCurrentItem()).thenReturn(item1); + when(mappingContext.isLeader()).thenReturn(true); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + LogicQueueMappingItem item2 = createLogicQueueMappingItem("BrokerA", 0, 0L, 0L); + items.add(item2); + QueryConsumerOffsetResponseHeader queryConsumerOffsetResponseHeader = new QueryConsumerOffsetResponseHeader(); + queryConsumerOffsetResponseHeader.setOffset(0L); + RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null); + when(responseFuture.get()).thenReturn(rpcResponse); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + queryConsumerOffsetResponseHeader.setOffset(-1L); + rpcResponse = new RpcResponse(ResponseCode.SUCCESS,queryConsumerOffsetResponseHeader,null); + when(responseFuture.get()).thenReturn(rpcResponse); + response = consumerManageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND); + } + + @Test + public void testRewriteRequestForStaticTopic() throws RpcException, ExecutionException, InterruptedException { + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setQueueId(0); + requestHeader.setCommitOffset(0L); + + RemotingCommand response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.NOT_LEADER_FOR_QUEUE); + + List items = new ArrayList<>(); + LogicQueueMappingItem item = createLogicQueueMappingItem("BrokerC", 0, 0L, 0L); + items.add(item); + when(mappingContext.getMappingItemList()).thenReturn(items); + when(mappingContext.isLeader()).thenReturn(true); + RpcResponse rpcResponse = new RpcResponse(ResponseCode.SUCCESS,new UpdateConsumerOffsetResponseHeader(),null); + when(responseFuture.get()).thenReturn(rpcResponse); + response = consumerManageProcessor.rewriteRequestForStaticTopic(requestHeader, mappingContext); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + public RemotingCommand buildQueryConsumerOffsetRequest(String group, String topic, int queueId,boolean setZeroIfNotFound) { + QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + requestHeader.setSetZeroIfNotFound(setZeroIfNotFound); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } + + public LogicQueueMappingItem createLogicQueueMappingItem(String brokerName, int queueId, long startOffset, long logicOffset) { + LogicQueueMappingItem item = new LogicQueueMappingItem(); + item.setBname(brokerName); + item.setQueueId(queueId); + item.setStartOffset(startOffset); + item.setLogicOffset(logicOffset); + return item; + } + private RemotingCommand buildUpdateConsumerOffsetRequest(String group, String topic, int queueId, long offset) { UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); requestHeader.setConsumerGroup(group); From d9d53d58cf7f32143485f12d4851d5c119d0855a Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+tanXiang003@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:22:34 +0800 Subject: [PATCH 065/265] add some tests for nameserver (#8349) --- .../namesrv/route/ZoneRouteRPCHook.java | 7 +- .../processor/RequestProcessorTest.java | 36 ++++ .../namesrv/route/ZoneRouteRPCHookTest.java | 164 ++++++++++++++++++ .../routeinfo/RouteInfoManagerNewTest.java | 69 +++++++- 4 files changed, 269 insertions(+), 7 deletions(-) create mode 100644 namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java index 4983c88c8af..a740a0f1b4e 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHook.java @@ -56,7 +56,6 @@ public void doAfterResponse(String remoteAddr, RemotingCommand request, Remoting return; } TopicRouteData topicRouteData = RemotingSerializable.decode(response.getBody(), TopicRouteData.class); - response.setBody(filterByZoneName(topicRouteData, zoneName).encode()); } @@ -64,6 +63,9 @@ private TopicRouteData filterByZoneName(TopicRouteData topicRouteData, String zo List brokerDataReserved = new ArrayList<>(); Map brokerDataRemoved = new HashMap<>(); for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { + if (brokerData.getBrokerAddrs() == null) { + continue; + } //master down, consume from slave. break nearby route rule. if (brokerData.getBrokerAddrs().get(MixAll.MASTER_ID) == null || StringUtils.equalsIgnoreCase(brokerData.getZoneName(), zoneName)) { @@ -85,9 +87,6 @@ private TopicRouteData filterByZoneName(TopicRouteData topicRouteData, String zo if (topicRouteData.getFilterServerTable() != null && !topicRouteData.getFilterServerTable().isEmpty()) { for (Entry entry : brokerDataRemoved.entrySet()) { BrokerData brokerData = entry.getValue(); - if (brokerData.getBrokerAddrs() == null) { - continue; - } brokerData.getBrokerAddrs().values() .forEach(brokerAddr -> topicRouteData.getFilterServerTable().remove(brokerAddr)); } diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java index 2b2cf629494..831558a0f68 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java @@ -448,6 +448,42 @@ public void testGetBrokerClusterInfo() throws RemotingCommandException { assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); } + @Test + public void testQueryDataVersion()throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.QUERY_DATA_VERSION); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetBrokerMemberBroker() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_BROKER_MEMBER_GROUP); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testBrokerHeartBeat() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.BROKER_HEARTBEAT); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testAddWritePermOfBroker() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.ADD_WRITE_PERM_OF_BROKER); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + @Test public void testWipeWritePermOfBroker() throws RemotingCommandException { ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java new file mode 100644 index 00000000000..1bf4a6c677f --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.namesrv.route; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + + +public class ZoneRouteRPCHookTest { + + private ZoneRouteRPCHook zoneRouteRPCHook; + + @Before + public void setup() { + zoneRouteRPCHook = new ZoneRouteRPCHook(); + } + + @Test + public void testDoAfterResponseWithNoZoneMode() { + RemotingCommand request1 = RemotingCommand.createRequestCommand(106,null); + zoneRouteRPCHook.doAfterResponse("", request1, null); + + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, "false"); + RemotingCommand request = RemotingCommand.createRequestCommand(105,null); + request.setExtFields(extFields); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(createSampleTopicRouteData())); + zoneRouteRPCHook.doAfterResponse("", request, response); + } + + @Test + public void testDoAfterResponseWithNoZoneName() { + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, "true"); + RemotingCommand request = RemotingCommand.createRequestCommand(105,null); + request.setExtFields(extFields); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(createSampleTopicRouteData())); + zoneRouteRPCHook.doAfterResponse("", request, response); + } + + @Test + public void testDoAfterResponseWithNoResponse() { + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, "true"); + RemotingCommand request = RemotingCommand.createRequestCommand(105,null); + request.setExtFields(extFields); + zoneRouteRPCHook.doAfterResponse("", request, null); + + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + zoneRouteRPCHook.doAfterResponse("", request, response); + + response.setBody(RemotingSerializable.encode(createSampleTopicRouteData())); + response.setCode(ResponseCode.NO_PERMISSION); + zoneRouteRPCHook.doAfterResponse("", request, response); + } + + + @Test + public void testDoAfterResponseWithValidZoneFiltering() throws Exception { + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, "true"); + extFields.put(MixAll.ZONE_NAME,"zone1"); + RemotingCommand request = RemotingCommand.createRequestCommand(105,null); + request.setExtFields(extFields); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + TopicRouteData topicRouteData = createSampleTopicRouteData(); + response.setBody(RemotingSerializable.encode(topicRouteData)); + zoneRouteRPCHook.doAfterResponse("", request, response); + + HashMap brokeraddrs = new HashMap<>(); + brokeraddrs.put(MixAll.MASTER_ID,"127.0.0.1:10911"); + topicRouteData.getBrokerDatas().get(0).setBrokerAddrs(brokeraddrs); + response.setBody(RemotingSerializable.encode(topicRouteData)); + zoneRouteRPCHook.doAfterResponse("", request, response); + + topicRouteData.getQueueDatas().add(createQueueData("BrokerB")); + HashMap brokeraddrsB = new HashMap<>(); + brokeraddrsB.put(MixAll.MASTER_ID,"127.0.0.1:10912"); + BrokerData brokerData1 = createBrokerData("BrokerB","zone2",brokeraddrsB); + BrokerData brokerData2 = createBrokerData("BrokerC","zone1",null); + topicRouteData.getBrokerDatas().add(brokerData1); + topicRouteData.getBrokerDatas().add(brokerData2); + response.setBody(RemotingSerializable.encode(topicRouteData)); + zoneRouteRPCHook.doAfterResponse("", request, response); + + topicRouteData.getFilterServerTable().put("127.0.0.1:10911",new ArrayList<>()); + response.setBody(RemotingSerializable.encode(topicRouteData)); + zoneRouteRPCHook.doAfterResponse("", request, response); + Assert.assertEquals(1,RemotingSerializable + .decode(response.getBody(), TopicRouteData.class) + .getFilterServerTable() + .size()); + + topicRouteData.getFilterServerTable().put("127.0.0.1:10912",new ArrayList<>()); + response.setBody(RemotingSerializable.encode(topicRouteData)); + zoneRouteRPCHook.doAfterResponse("", request, response); + Assert.assertEquals(1,RemotingSerializable + .decode(response.getBody(), TopicRouteData.class) + .getFilterServerTable() + .size()); + } + + private TopicRouteData createSampleTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + List brokerDatas = new ArrayList<>(); + BrokerData brokerData = createBrokerData("BrokerA","zone1",new HashMap<>()); + List queueDatas = new ArrayList<>(); + QueueData queueData = createQueueData("BrokerA"); + queueDatas.add(queueData); + brokerDatas.add(brokerData); + topicRouteData.setBrokerDatas(brokerDatas); + topicRouteData.setQueueDatas(queueDatas); + return topicRouteData; + } + + private BrokerData createBrokerData(String brokerName,String zoneName,HashMap brokerAddrs) { + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName(brokerName); + brokerData.setZoneName(zoneName); + brokerData.setBrokerAddrs(brokerAddrs); + return brokerData; + } + + private QueueData createQueueData(String brokerName) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(brokerName); + queueData.setReadQueueNums(8); + queueData.setWriteQueueNums(8); + queueData.setPerm(6); + return queueData; + } +} diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java index b52cf50740a..5e58cfc124e 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerNewTest.java @@ -416,9 +416,12 @@ public void pickupTopicRouteDataWithSlave() { } @Test - public void scanNotActiveBroker() { + public void scanNotActiveBroker() throws InterruptedException { registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); routeInfoManager.scanNotActiveBroker(); + registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo.defaultBroker(),"TestTopic"); + Thread.sleep(30000); + routeInfoManager.scanNotActiveBroker(); } @Test @@ -589,6 +592,16 @@ public void onChannelDestroy() { assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); } + @Test + public void onChannelDestroyByBrokerInfo() { + registerBroker(BrokerBasicInfo.defaultBroker(), mock(Channel.class), null, "TestTopic", "TestTopic1"); + BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(DEFAULT_CLUSTER, DEFAULT_ADDR); + routeInfoManager.onChannelDestroy(brokerAddrInfo); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull(); + } + @Test public void switchBrokerRole_ChannelDestroy() { final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false); @@ -728,6 +741,23 @@ private RegisterBrokerResult registerBrokerWithNormalTopic(BrokerBasicInfo broke return registerBroker(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics); } + private RegisterBrokerResult registerBrokerWithNormalTopicAndExpire(BrokerBasicInfo brokerInfo, String... topics) { + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + TopicConfig baseTopic = new TopicConfig("baseTopic"); + topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic); + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + + return registerBrokerWithExpiredTime(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics); + } + private RegisterBrokerResult registerBrokerWithOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) { ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); @@ -785,7 +815,7 @@ private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion); topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); - RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker( + return routeInfoManager.registerBroker( brokerInfo.clusterName, brokerInfo.brokerAddr, brokerInfo.brokerName, @@ -795,7 +825,40 @@ private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel null, brokerInfo.enableActingMaster, topicConfigSerializeWrapper, new ArrayList<>(), channel); - return registerBrokerResult; + } + + private RegisterBrokerResult registerBrokerWithExpiredTime(BrokerBasicInfo brokerInfo, Channel channel, + ConcurrentMap topicConfigConcurrentHashMap, String... topics) { + + if (topicConfigConcurrentHashMap == null) { + topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + TopicConfig baseTopic = new TopicConfig("baseTopic"); + topicConfigConcurrentHashMap.put(baseTopic.getTopicName(), baseTopic); + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + } + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); + + return routeInfoManager.registerBroker( + brokerInfo.clusterName, + brokerInfo.brokerAddr, + brokerInfo.brokerName, + brokerInfo.brokerId, + brokerInfo.haAddr, + "", + 30000L, + brokerInfo.enableActingMaster, + topicConfigSerializeWrapper, new ArrayList<>(), channel); } private void registerSingleTopicWithBrokerName(String brokerName, String... topics) { From bd61774b1fc19f92c8c3a466420d50899ed61d9b Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 22 Jul 2024 10:05:14 +0800 Subject: [PATCH 066/265] [ISSUE #8411] Add more test coverage for DefaultMQPushConsumerImpl (#8412) * [ISSUE #8411] Add more test coverage for DefaultMQPushConsumerImpl * Update * Update --- .../DefaultMQPushConsumerImplTest.java | 736 +++++++++++++++++- 1 file changed, 719 insertions(+), 17 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java index 879bbc593c1..68563c02562 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java @@ -17,17 +17,50 @@ package org.apache.rocketmq.client.impl.consumer; -import java.util.List; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.consumer.AckCallback; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.PopCallback; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageContext; import org.apache.rocketmq.client.hook.FilterMessageHook; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.stat.ConsumerStatsManager; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ServiceState; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.body.ConsumeStatus; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Assert; +import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -36,17 +69,85 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class DefaultMQPushConsumerImplTest { + @Mock private DefaultMQPushConsumer defaultMQPushConsumer; + @Mock + private MQClientInstance mQClientFactory; + + @Mock + private RebalanceImpl rebalanceImpl; + + @Mock + private PullAPIWrapper pullAPIWrapper; + + @Mock + private PullRequest pullRequest; + + @Mock + private PopRequest popRequest; + + @Mock + private ProcessQueue processQueue; + + @Mock + private PopProcessQueue popProcessQueue; + + @Mock + private MQClientAPIImpl mqClientAPIImpl; + + @Mock + private OffsetStore offsetStore; + + private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + @Rule public ExpectedException thrown = ExpectedException.none(); + private final String defaultKey = "defaultKey"; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final String defaultGroup = "defaultGroup"; + + private final long defaultTimeout = 3000L; @Test public void checkConfigTest() throws MQClientException { @@ -62,19 +163,14 @@ public void checkConfigTest() throws MQClientException { consumer.setConsumeThreadMin(10); consumer.setConsumeThreadMax(9); - consumer.registerMessageListener(new MessageListenerConcurrently() { - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } - }); + consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> ConsumeConcurrentlyStatus.CONSUME_SUCCESS); DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(consumer, null); defaultMQPushConsumerImpl.start(); } @Test - public void testHook() throws Exception { + public void testHook() { DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null); defaultMQPushConsumerImpl.registerConsumeMessageHook(new ConsumeMessageHook() { @Override @@ -110,14 +206,10 @@ public void filterMessage(FilterMessageContext context) { @Ignore @Test public void testPush() throws Exception { - when(defaultMQPushConsumer.getMessageListener()).thenReturn(new MessageListenerConcurrently() { - @Override - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { - assertThat(msgs).size().isGreaterThan(0); - assertThat(context).isNotNull(); - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - } + when(defaultMQPushConsumer.getMessageListener()).thenReturn((MessageListenerConcurrently) (msgs, context) -> { + assertThat(msgs).size().isGreaterThan(0); + assertThat(context).isNotNull(); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; }); DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, null); try { @@ -126,4 +218,614 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, defaultMQPushConsumerImpl.shutdown(); } } + + @Before + public void init() throws NoSuchFieldException, IllegalAccessException { + MQAdminImpl mqAdminImpl = mock(MQAdminImpl.class); + when(mQClientFactory.getMQAdminImpl()).thenReturn(mqAdminImpl); + ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class); + ConsumeStatus consumeStatus = mock(ConsumeStatus.class); + when(consumerStatsManager.consumeStatus(any(), any())).thenReturn(consumeStatus); + when(mQClientFactory.getConsumerStatsManager()).thenReturn(consumerStatsManager); + when(mQClientFactory.getPullMessageService()).thenReturn(mock(PullMessageService.class)); + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl); + FindBrokerResult findBrokerResult = mock(FindBrokerResult.class); + when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr); + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(findBrokerResult); + Set messageQueueSet = Collections.singleton(createMessageQueue()); + ConcurrentMap> topicMessageQueueMap = new ConcurrentHashMap<>(); + topicMessageQueueMap.put(defaultTopic, messageQueueSet); + when(rebalanceImpl.getTopicSubscribeInfoTable()).thenReturn(topicMessageQueueMap); + ConcurrentMap processQueueTable = new ConcurrentHashMap<>(); + when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueTable); + RPCHook rpcHook = mock(RPCHook.class); + defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(defaultMQPushConsumer, rpcHook); + defaultMQPushConsumerImpl.setOffsetStore(offsetStore); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "mQClientFactory", mQClientFactory, true); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "rebalanceImpl", rebalanceImpl, true); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "pullAPIWrapper", pullAPIWrapper, true); + FilterMessageHook filterMessageHook = mock(FilterMessageHook.class); + ArrayList filterMessageHookList = new ArrayList<>(); + filterMessageHookList.add(filterMessageHook); + ConsumeMessageService consumeMessagePopService = mock(ConsumeMessageService.class); + ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "filterMessageHookList", filterMessageHookList, true); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "consumeMessageService", consumeMessageService, true); + FieldUtils.writeDeclaredField(defaultMQPushConsumerImpl, "consumeMessagePopService", consumeMessagePopService, true); + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTopic(defaultTopic); + subscriptionDataMap.put(defaultTopic, subscriptionData); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + } + + @Test + public void testFetchSubscribeMessageQueues() throws MQClientException { + Set actual = defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(defaultTopic); + assertNotNull(actual); + Assert.assertEquals(1, actual.size()); + MessageQueue next = actual.iterator().next(); + assertEquals(defaultTopic, next.getTopic()); + assertEquals(defaultBroker, next.getBrokerName()); + assertEquals(0, next.getQueueId()); + } + + @Test + public void testEarliestMsgStoreTime() throws MQClientException { + assertEquals(0, defaultMQPushConsumerImpl.earliestMsgStoreTime(createMessageQueue())); + } + + @Test + public void testMaxOffset() throws MQClientException { + assertEquals(0, defaultMQPushConsumerImpl.maxOffset(createMessageQueue())); + } + + @Test + public void testMinOffset() throws MQClientException { + assertEquals(0, defaultMQPushConsumerImpl.minOffset(createMessageQueue())); + } + + @Test + public void testGetOffsetStore() { + assertEquals(offsetStore, defaultMQPushConsumerImpl.getOffsetStore()); + } + + @Test + public void testPullMessageWithStateNotOk() { + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithIsPause() { + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.setPause(true); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithMsgCountFlowControl() { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + TreeMap treeMap = new TreeMap<>(); + treeMap.put(1L, new MessageExt()); + when(processQueue.getMsgTreeMap()).thenReturn(treeMap); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(1); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithMsgSizeFlowControl() { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + TreeMap treeMap = new TreeMap<>(); + treeMap.put(1L, new MessageExt()); + when(processQueue.getMsgTreeMap()).thenReturn(treeMap); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(1); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithMaxSpanFlowControl() { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMaxSpan()).thenReturn(2L); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + TreeMap treeMap = new TreeMap<>(); + treeMap.put(1L, new MessageExt()); + when(processQueue.getMsgTreeMap()).thenReturn(treeMap); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithNotLocked() { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.setConsumeOrderly(true); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithSubscriptionDataIsNull() { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithNoMatchedMsg() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + PullResult pullResultMock = mock(PullResult.class); + when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock); + when(pullResultMock.getPullStatus()).thenReturn(PullStatus.NO_MATCHED_MSG); + doAnswer(invocation -> { + PullCallback callback = invocation.getArgument(12); + PullResult pullResult = mock(PullResult.class); + callback.onSuccess(pullResult); + return null; + }).when(pullAPIWrapper).pullKernelImpl( + any(MessageQueue.class), + any(), + any(), + anyLong(), + anyLong(), + anyInt(), + anyInt(), + anyInt(), + anyLong(), + anyLong(), + anyLong(), + any(CommunicationMode.class), + any(PullCallback.class)); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithOffsetIllegal() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + PullResult pullResultMock = mock(PullResult.class); + when(pullAPIWrapper.processPullResult(any(MessageQueue.class), any(PullResult.class), any(SubscriptionData.class))).thenReturn(pullResultMock); + when(pullResultMock.getPullStatus()).thenReturn(PullStatus.OFFSET_ILLEGAL); + doAnswer(invocation -> { + PullCallback callback = invocation.getArgument(12); + PullResult pullResult = mock(PullResult.class); + callback.onSuccess(pullResult); + return null; + }).when(pullAPIWrapper).pullKernelImpl( + any(MessageQueue.class), + any(), + any(), + anyLong(), + anyLong(), + anyInt(), + anyInt(), + anyInt(), + anyLong(), + anyLong(), + anyLong(), + any(CommunicationMode.class), + any(PullCallback.class)); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPullMessageWithException() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + when(processQueue.getMsgCount()).thenReturn(new AtomicLong(2)); + when(processQueue.getMsgSize()).thenReturn(new AtomicLong(3 * 1024 * 1024)); + when(pullRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(pullRequest.getProcessQueue()).thenReturn(processQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPullThresholdForQueue()).thenReturn(3); + when(defaultMQPushConsumer.getPullThresholdSizeForQueue()).thenReturn(10); + doAnswer(invocation -> { + PullCallback callback = invocation.getArgument(12); + callback.onException(new RuntimeException("exception")); + return null; + }).when(pullAPIWrapper).pullKernelImpl( + any(MessageQueue.class), + any(), + any(), + anyLong(), + anyLong(), + anyInt(), + anyInt(), + anyInt(), + anyLong(), + anyLong(), + anyLong(), + any(CommunicationMode.class), + any(PullCallback.class)); + defaultMQPushConsumerImpl.pullMessage(pullRequest); + } + + @Test + public void testPopMessageWithFound() throws RemotingException, InterruptedException, MQClientException { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(popRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTagsSet(Collections.singleton("*")); + subscriptionDataMap.put(defaultTopic, subscriptionData); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + doAnswer(invocation -> { + PopCallback callback = invocation.getArgument(5); + PopResult popResult = mock(PopResult.class); + when(popResult.getPopStatus()).thenReturn(PopStatus.FOUND); + when(popResult.getMsgFoundList()).thenReturn(Collections.singletonList(createMessageExt())); + callback.onSuccess(popResult); + return null; + }).when(pullAPIWrapper).popAsync( + any(MessageQueue.class), + anyLong(), + anyInt(), + any(), + anyLong(), + any(PopCallback.class), + anyBoolean(), + anyInt(), + anyBoolean(), + any(), + any()); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithException() throws RemotingException, InterruptedException, MQClientException { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(popRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTagsSet(Collections.singleton("*")); + subscriptionDataMap.put(defaultTopic, subscriptionData); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + doAnswer(invocation -> { + PopCallback callback = invocation.getArgument(5); + callback.onException(new RuntimeException("exception")); + return null; + }).when(pullAPIWrapper).popAsync( + any(MessageQueue.class), + anyLong(), + anyInt(), + any(), + anyLong(), + any(PopCallback.class), + anyBoolean(), + anyInt(), + anyBoolean(), + any(), + any()); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithNoNewMsg() throws RemotingException, InterruptedException, MQClientException { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(popRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTagsSet(Collections.singleton("*")); + subscriptionDataMap.put(defaultTopic, subscriptionData); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + doAnswer(invocation -> { + PopCallback callback = invocation.getArgument(5); + PopResult popResult = mock(PopResult.class); + when(popResult.getPopStatus()).thenReturn(PopStatus.NO_NEW_MSG); + callback.onSuccess(popResult); + return null; + }).when(pullAPIWrapper).popAsync( + any(MessageQueue.class), + anyLong(), + anyInt(), + any(), + anyLong(), + any(PopCallback.class), + anyBoolean(), + anyInt(), + anyBoolean(), + any(), + any()); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithPollingFull() throws RemotingException, InterruptedException, MQClientException { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(popRequest.getMessageQueue()).thenReturn(createMessageQueue()); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTagsSet(Collections.singleton("*")); + subscriptionDataMap.put(defaultTopic, subscriptionData); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + doAnswer(invocation -> { + PopCallback callback = invocation.getArgument(5); + PopResult popResult = mock(PopResult.class); + when(popResult.getPopStatus()).thenReturn(PopStatus.POLLING_FULL); + callback.onSuccess(popResult); + return null; + }).when(pullAPIWrapper).popAsync(any( + MessageQueue.class), + anyLong(), + anyInt(), + any(), + anyLong(), + any(PopCallback.class), + anyBoolean(), + anyInt(), + anyBoolean(), + any(), + any()); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithStateNotOk() { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithIsPause() { + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.setPause(true); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithWaiAckMsgCountFlowControl() { + when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2); + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(1); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.popMessage(popRequest); + } + + @Test + public void testPopMessageWithSubscriptionDataIsNull() throws RemotingException, InterruptedException, MQClientException { + when(popProcessQueue.getWaiAckMsgCount()).thenReturn(2); + when(popRequest.getPopProcessQueue()).thenReturn(popProcessQueue); + when(popRequest.getMessageQueue()).thenReturn(createMessageQueue()); + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + when(defaultMQPushConsumer.getPopThresholdForQueue()).thenReturn(3); + defaultMQPushConsumerImpl.popMessage(popRequest); + verify(pullAPIWrapper).popAsync(any(MessageQueue.class), + eq(60000L), + eq(0), + any(), + eq(15000L), + any(PopCallback.class), + eq(true), + eq(0), + eq(false), + any(), + any()); + } + + @Test + public void testQueryMessage() throws InterruptedException, MQClientException { + assertNull(defaultMQPushConsumerImpl.queryMessage(defaultTopic, defaultKey, 1, 0, 1)); + } + + @Test + public void testQueryMessageByUniqKey() throws InterruptedException, MQClientException { + assertNull(defaultMQPushConsumerImpl.queryMessageByUniqKey(defaultTopic, defaultKey)); + } + + @Test + public void testSendMessageBack() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + defaultMQPushConsumerImpl.sendMessageBack(createMessageExt(), 1, createMessageQueue()); + verify(mqClientAPIImpl).consumerSendMessageBack( + eq(defaultBrokerAddr), + any(), + any(MessageExt.class), + any(), + eq(1), + eq(5000L), + eq(0)); + } + + @Test + public void testAckAsync() throws MQBrokerException, RemotingException, InterruptedException { + doAnswer(invocation -> { + AckCallback callback = invocation.getArgument(2); + AckResult result = mock(AckResult.class); + when(result.getStatus()).thenReturn(AckStatus.OK); + callback.onSuccess(result); + return null; + }).when(mqClientAPIImpl).ackMessageAsync(any(), + anyLong(), + any(AckCallback.class), + any(AckMessageRequestHeader.class)); + defaultMQPushConsumerImpl.ackAsync(createMessageExt(), defaultGroup); + verify(mqClientAPIImpl).ackMessageAsync(eq(defaultBrokerAddr), + eq(3000L), + any(AckCallback.class), + any(AckMessageRequestHeader.class)); + } + + @Test + public void testChangePopInvisibleTimeAsync() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + AckCallback callback = mock(AckCallback.class); + String extraInfo = createMessageExt().getProperty(MessageConst.PROPERTY_POP_CK); + defaultMQPushConsumerImpl.changePopInvisibleTimeAsync(defaultTopic, defaultGroup, extraInfo, defaultTimeout, callback); + verify(mqClientAPIImpl).changeInvisibleTimeAsync(eq(defaultBroker), + eq(defaultBrokerAddr), + any(ChangeInvisibleTimeRequestHeader.class), + eq(defaultTimeout), + any(AckCallback.class)); + } + + @Test + public void testShutdown() { + defaultMQPushConsumerImpl.setServiceState(ServiceState.RUNNING); + defaultMQPushConsumerImpl.shutdown(); + assertEquals(ServiceState.SHUTDOWN_ALREADY, defaultMQPushConsumerImpl.getServiceState()); + } + + @Test + public void testSubscribe() throws MQClientException { + defaultMQPushConsumerImpl.subscribe(defaultTopic, "fullClassname", "filterClassSource"); + RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl(); + assertEquals(1, actual.getSubscriptionInner().size()); + } + + @Test + public void testSubscribeByMessageSelector() throws MQClientException { + MessageSelector messageSelector = mock(MessageSelector.class); + defaultMQPushConsumerImpl.subscribe(defaultTopic, messageSelector); + RebalanceImpl actual = defaultMQPushConsumerImpl.getRebalanceImpl(); + assertEquals(1, actual.getSubscriptionInner().size()); + } + + @Test + public void testSuspend() { + defaultMQPushConsumerImpl.suspend(); + assertTrue(defaultMQPushConsumerImpl.isPause()); + } + + @Test + public void testViewMessage() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + assertNull(defaultMQPushConsumerImpl.viewMessage(defaultTopic, createMessageExt().getMsgId())); + } + + @Test + public void testResetOffsetByTimeStamp() throws MQClientException { + ConcurrentMap subscriptionDataMap = new ConcurrentHashMap<>(); + subscriptionDataMap.put(defaultTopic, new SubscriptionData()); + when(rebalanceImpl.getSubscriptionInner()).thenReturn(subscriptionDataMap); + defaultMQPushConsumerImpl.resetOffsetByTimeStamp(System.currentTimeMillis()); + verify(mQClientFactory).resetOffset(eq(defaultTopic), any(), any()); + } + + @Test + public void testSearchOffset() throws MQClientException { + assertEquals(0, defaultMQPushConsumerImpl.searchOffset(createMessageQueue(), System.currentTimeMillis())); + } + + @Test + public void testQueryConsumeTimeSpan() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.getBrokerDatas().add(createBrokerData()); + when(mqClientAPIImpl.getTopicRouteInfoFromNameServer(any(), anyLong())).thenReturn(topicRouteData); + List actual = defaultMQPushConsumerImpl.queryConsumeTimeSpan(defaultTopic); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testTryResetPopRetryTopic() { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.getBrokerDatas().add(createBrokerData()); + MessageExt messageExt = createMessageExt(); + List msgs = new ArrayList<>(); + messageExt.setTopic(MixAll.RETRY_GROUP_TOPIC_PREFIX + defaultGroup + "_" + defaultTopic); + msgs.add(messageExt); + defaultMQPushConsumerImpl.tryResetPopRetryTopic(msgs, defaultGroup); + assertEquals(defaultTopic, msgs.get(0).getTopic()); + } + + @Test + public void testGetPopDelayLevel() { + int[] actual = defaultMQPushConsumerImpl.getPopDelayLevel(); + int[] expected = new int[]{10, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200}; + assertArrayEquals(expected, actual); + } + + @Test + public void testGetMessageQueueListener() { + assertNull(defaultMQPushConsumerImpl.getMessageQueueListener()); + } + + @Test + public void testConsumerRunningInfo() { + ConcurrentMap processQueueMap = new ConcurrentHashMap<>(); + ConcurrentMap popProcessQueueMap = new ConcurrentHashMap<>(); + processQueueMap.put(createMessageQueue(), new ProcessQueue()); + popProcessQueueMap.put(createMessageQueue(), new PopProcessQueue()); + when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap); + when(rebalanceImpl.getPopProcessQueueTable()).thenReturn(popProcessQueueMap); + ConsumerRunningInfo actual = defaultMQPushConsumerImpl.consumerRunningInfo(); + assertNotNull(actual); + assertEquals(1, actual.getSubscriptionSet().size()); + assertEquals(defaultTopic, actual.getSubscriptionSet().iterator().next().getTopic()); + assertEquals(1, actual.getMqTable().size()); + assertEquals(1, actual.getMqPopTable().size()); + assertEquals(1, actual.getStatusTable().size()); + } + + private BrokerData createBrokerData() { + BrokerData result = new BrokerData(); + HashMap brokerAddrMap = new HashMap<>(); + brokerAddrMap.put(MixAll.MASTER_ID, defaultBrokerAddr); + result.setBrokerAddrs(brokerAddrMap); + result.setBrokerName(defaultBroker); + return result; + } + + private MessageQueue createMessageQueue() { + MessageQueue result = new MessageQueue(); + result.setQueueId(0); + result.setBrokerName(defaultBroker); + result.setTopic(defaultTopic); + return result; + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + long curTime = System.currentTimeMillis(); + result.setBornTimestamp(curTime - 1000); + String popProps = String.format("%d %d %d %d %d %s %d %d %d", curTime, curTime, curTime, curTime, curTime, defaultBroker, 1, 0L, 1L); + result.getProperties().put(MessageConst.PROPERTY_POP_CK, popProps); + result.setKeys("keys"); + result.setTags("*"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } } From 6ecdc482232918a3b2a8b0fd36cb1d9a03a60415 Mon Sep 17 00:00:00 2001 From: yueran Date: Mon, 22 Jul 2024 11:14:35 +0800 Subject: [PATCH 067/265] [ISSUE #8413] Add some test cases for commom module --- .../common/BrokerConfigSingletonTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java new file mode 100644 index 00000000000..b98a6e37e69 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/BrokerConfigSingletonTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +import org.junit.Assert; +import org.junit.Test; + +public class BrokerConfigSingletonTest { + + /** + * Tests the behavior of getting the broker configuration when it has not been initialized. + * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be obtained without initialization. + */ + @Test(expected = IllegalArgumentException.class) + public void getBrokerConfig_NullConfiguration_ThrowsException() { + BrokerConfigSingleton.getBrokerConfig(); + } + + /** + * Tests the behavior of setting the broker configuration after it has already been initialized. + * Expects an IllegalArgumentException to be thrown, ensuring that the configuration cannot be reset once set. + * Also asserts that the returned brokerConfig instance is the same as the one set, confirming the singleton property. + */ + @Test(expected = IllegalArgumentException.class) + public void setBrokerConfig_AlreadyInitialized_ThrowsException() { + BrokerConfig config = new BrokerConfig(); + BrokerConfigSingleton.setBrokerConfig(config); + Assert.assertSame("Expected brokerConfig instance is not returned", config, BrokerConfigSingleton.getBrokerConfig()); + BrokerConfigSingleton.setBrokerConfig(config); + } + +} From 86d59d2485b5fed162db3743e11c0902de3e34ad Mon Sep 17 00:00:00 2001 From: rongtong Date: Mon, 22 Jul 2024 17:20:11 +0800 Subject: [PATCH 068/265] Add the ability to write ConsumeQueue using fileChannel to prevent JVM crashes in some situations (#8403) --- .../apache/rocketmq/store/ConsumeQueue.java | 15 +++++++++-- .../store/config/MessageStoreConfig.java | 10 +++++++ .../store/logfile/DefaultMappedFile.java | 27 +++++++++++++++---- .../rocketmq/store/logfile/MappedFile.java | 11 ++++++++ .../store/queue/BatchConsumeQueue.java | 7 ++++- .../store/queue/SparseConsumeQueue.java | 10 ++++++- 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index 569cc3cfaa6..eb8af4ab190 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -833,7 +833,13 @@ private boolean putMessagePositionInfo(final long offset, final int size, final } } this.setMaxPhysicOffset(offset + size); - return mappedFile.appendMessage(this.byteBufferIndex.array()); + boolean appendResult; + if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) { + appendResult = mappedFile.appendMessageUsingFileChannel(this.byteBufferIndex.array()); + } else { + appendResult = mappedFile.appendMessage(this.byteBufferIndex.array()); + } + return appendResult; } return false; } @@ -846,7 +852,12 @@ private void fillPreBlank(final MappedFile mappedFile, final long untilWhere) { int until = (int) (untilWhere % this.mappedFileQueue.getMappedFileSize()); for (int i = 0; i < until; i += CQ_STORE_UNIT_SIZE) { - mappedFile.appendMessage(byteBuffer.array()); + if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) { + mappedFile.appendMessageUsingFileChannel(byteBuffer.array()); + } else { + mappedFile.appendMessage(byteBuffer.array()); + } + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0060b144cff..5b2a1931b3b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -419,6 +419,8 @@ public class MessageStoreConfig { */ private boolean readUnCommitted = false; + private boolean putConsumeQueueDataByFileChannel = true; + public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } @@ -1832,4 +1834,12 @@ public boolean isReadUnCommitted() { public void setReadUnCommitted(boolean readUnCommitted) { this.readUnCommitted = readUnCommitted; } + + public boolean isPutConsumeQueueDataByFileChannel() { + return putConsumeQueueDataByFileChannel; + } + + public void setPutConsumeQueueDataByFileChannel(boolean putConsumeQueueDataByFileChannel) { + this.putConsumeQueueDataByFileChannel = putConsumeQueueDataByFileChannel; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java index 03477c33249..c490d093a16 100644 --- a/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java +++ b/store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java @@ -97,14 +97,14 @@ public class DefaultMappedFile extends AbstractMappedFile { protected long mappedByteBufferAccessCountSinceLastSwap = 0L; /** - * If this mapped file belongs to consume queue, this field stores store-timestamp of first message referenced - * by this logical queue. + * If this mapped file belongs to consume queue, this field stores store-timestamp of first message referenced by + * this logical queue. */ private long startTimestamp = -1; /** - * If this mapped file belongs to consume queue, this field stores store-timestamp of last message referenced - * by this logical queue. + * If this mapped file belongs to consume queue, this field stores store-timestamp of last message referenced by + * this logical queue. */ private long stopTimestamp = -1; @@ -357,6 +357,24 @@ public boolean appendMessage(final byte[] data, final int offset, final int leng return false; } + @Override + public boolean appendMessageUsingFileChannel(byte[] data) { + int currentPos = WROTE_POSITION_UPDATER.get(this); + + if ((currentPos + data.length) <= this.fileSize) { + try { + this.fileChannel.position(currentPos); + this.fileChannel.write(ByteBuffer.wrap(data, 0, data.length)); + } catch (Throwable e) { + log.error("Error occurred when append message to mappedFile.", e); + } + WROTE_POSITION_UPDATER.addAndGet(this, data.length); + return true; + } + + return false; + } + /** * @return The current flushed position */ @@ -840,7 +858,6 @@ public void setStopTimestamp(long stopTimestamp) { this.stopTimestamp = stopTimestamp; } - public Iterator iterator(int startPos) { return new Itr(startPos); } diff --git a/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java b/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java index dfcf66f0882..fd70d6c5634 100644 --- a/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java +++ b/store/src/main/java/org/apache/rocketmq/store/logfile/MappedFile.java @@ -101,12 +101,23 @@ public interface MappedFile { /** * Appends a raw message data represents by a byte array to the current {@code MappedFile}. + * Using mappedByteBuffer * * @param data the byte array to append * @return true if success; false otherwise. */ boolean appendMessage(byte[] data); + + /** + * Appends a raw message data represents by a byte array to the current {@code MappedFile}. + * Using fileChannel + * + * @param data the byte array to append + * @return true if success; false otherwise. + */ + boolean appendMessageUsingFileChannel(byte[] data); + /** * Appends a raw message data represents by a byte array to the current {@code MappedFile}. * diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java index 7108c835c8e..16171827245 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java @@ -587,7 +587,12 @@ public boolean putBatchMessagePositionInfo(final long offset, final int size, fi MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(this.mappedFileQueue.getMaxOffset()); if (mappedFile != null) { boolean isNewFile = isNewFile(mappedFile); - boolean appendRes = mappedFile.appendMessage(this.byteBufferItem.array()); + boolean appendRes; + if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) { + appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array()); + } else { + appendRes = mappedFile.appendMessage(this.byteBufferItem.array()); + } if (appendRes) { maxMsgPhyOffsetInCommitLog = offset; maxOffsetInQueue = msgBaseOffset + batchSize; diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java index 4a5f3a93b1d..7e14de30abc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/SparseConsumeQueue.java @@ -262,7 +262,15 @@ public void putEndPositionInfo(MappedFile mappedFile) { this.byteBufferItem.putShort((short)0); this.byteBufferItem.putInt(INVALID_POS); this.byteBufferItem.putInt(0); // 4 bytes reserved - boolean appendRes = mappedFile.appendMessage(this.byteBufferItem.array()); + + boolean appendRes; + + if (messageStore.getMessageStoreConfig().isPutConsumeQueueDataByFileChannel()) { + appendRes = mappedFile.appendMessageUsingFileChannel(this.byteBufferItem.array()); + } else { + appendRes = mappedFile.appendMessage(this.byteBufferItem.array()); + } + if (!appendRes) { log.error("append end position info into {} failed", mappedFile.getFileName()); } From 8341c13d064c96e9ef70da4fcf17e49d3e1847f9 Mon Sep 17 00:00:00 2001 From: imzs Date: Tue, 23 Jul 2024 10:24:12 +0800 Subject: [PATCH 069/265] [ISSUE #8402] Fix init retry topic offset incorrect when EscapeBridge enabled (#8404) --- .../broker/processor/PopMessageProcessor.java | 10 +-- .../broker/processor/PopReviveService.java | 4 +- .../processor/PopMessageProcessorTest.java | 63 ++++++++++++++++++- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 0304a5dab08..89b4c39d72b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -723,8 +723,8 @@ private long getPopOffset(String topic, String group, int queueId, int initMode, private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) { long offset; - if (ConsumeInitMode.MIN == initMode) { - return this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); + if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); } else { if (this.brokerController.getBrokerConfig().isInitPopOffsetByCheckMsgInMem() && this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId) <= 0 && @@ -738,10 +738,10 @@ private long getInitOffset(String topic, String group, int queueId, int initMode offset = 0; } } - if (init) { - this.brokerController.getConsumerOffsetManager().commitOffset( + } + if (init) { // whichever initMode + this.brokerController.getConsumerOffsetManager().commitOffset( "getPopOffset", group, topic, queueId, offset); - } } return offset; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index e3ba492f280..8074af23bfe 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -125,7 +125,7 @@ private boolean reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt) msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime())); } msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - addRetryTopicIfNoExit(msgInner.getTopic(), popCheckPoint.getCId()); + addRetryTopicIfNotExist(msgInner.getTopic(), popCheckPoint.getCId()); PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); PopMetricsManager.incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus()); if (brokerController.getBrokerConfig().isEnablePopLog()) { @@ -153,7 +153,7 @@ private void initPopRetryOffset(String topic, String consumerGroup) { } } - private void addRetryTopicIfNoExit(String topic, String consumerGroup) { + public void addRetryTopicIfNotExist(String topic, String consumerGroup) { if (brokerController != null) { TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); if (topicConfig != null) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java index d8c8fa1034e..8a2ce8a2ba4 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.ConsumeInitMode; import org.apache.rocketmq.common.message.MessageDecoder; @@ -40,6 +41,7 @@ import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.logfile.DefaultMappedFile; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -53,6 +55,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -151,14 +154,70 @@ public void testProcessRequest_whenTimerWheelIsFalse() throws RemotingCommandExc assertThat(response.getRemark()).contains("pop message is forbidden because timerWheelEnable is false"); } + @Test + public void testGetInitOffset_retryTopic() throws RemotingCommandException { + when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); + String newGroup = group + "-" + System.currentTimeMillis(); + String retryTopic = KeyBuilder.buildPopRetryTopic(topic, newGroup); + long minOffset = 100L; + when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset); + brokerController.getTopicConfigManager().getTopicConfigTable().put(retryTopic, new TopicConfig(retryTopic, 1, 1)); + GetMessageResult getMessageResult = createGetMessageResult(0); + when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(getMessageResult)); + + long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0); + Assert.assertEquals(-1, offset); + + RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX); + popMessageProcessor.processRequest(handlerContext, request); + offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0); + Assert.assertEquals(minOffset, offset); + + when(messageStore.getMinOffsetInQueue(retryTopic, 0)).thenReturn(minOffset * 2); + popMessageProcessor.processRequest(handlerContext, request); + offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, retryTopic, 0); + Assert.assertEquals(minOffset, offset); // will not entry getInitOffset() again + messageStore.getMinOffsetInQueue(retryTopic, 0); // prevent UnnecessaryStubbingException + } + + @Test + public void testGetInitOffset_normalTopic() throws RemotingCommandException { + long maxOffset = 999L; + when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); + when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset); + String newGroup = group + "-" + System.currentTimeMillis(); + GetMessageResult getMessageResult = createGetMessageResult(0); + when(messageStore.getMessageAsync(eq(newGroup), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(getMessageResult)); + + long offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0); + Assert.assertEquals(-1, offset); + + RemotingCommand request = createPopMsgCommand(newGroup, topic, 0, ConsumeInitMode.MAX); + popMessageProcessor.processRequest(handlerContext, request); + offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0); + Assert.assertEquals(maxOffset - 1, offset); // checkInMem return false + + when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset * 2); + popMessageProcessor.processRequest(handlerContext, request); + offset = brokerController.getConsumerOffsetManager().queryOffset(newGroup, topic, 0); + Assert.assertEquals(maxOffset - 1, offset); // will not entry getInitOffset() again + messageStore.getMaxOffsetInQueue(topic, 0); // prevent UnnecessaryStubbingException + } + private RemotingCommand createPopMsgCommand() { + return createPopMsgCommand(group, topic, -1, ConsumeInitMode.MAX); + } + + private RemotingCommand createPopMsgCommand(String group, String topic, int queueId, int initMode) { PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); requestHeader.setConsumerGroup(group); requestHeader.setMaxMsgNums(30); - requestHeader.setQueueId(-1); + requestHeader.setQueueId(queueId); requestHeader.setTopic(topic); requestHeader.setInvisibleTime(10_000); - requestHeader.setInitMode(ConsumeInitMode.MAX); + requestHeader.setInitMode(initMode); requestHeader.setOrder(false); requestHeader.setPollTime(15_000); requestHeader.setBornTime(System.currentTimeMillis()); From 19ef75417751ee81f690c318895ad3c1c5143ce4 Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+TanXiang7o@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:26:21 +0800 Subject: [PATCH 070/265] [ISSUE #8421] Add more test coverage for SlaveSynchronize (#8422) --- .../broker/slave/SlaveSynchronizeTest.java | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java new file mode 100644 index 00000000000..95db733d0d1 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/slave/SlaveSynchronizeTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.slave; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.out.BrokerOuterAPI; +import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor; +import org.apache.rocketmq.broker.schedule.ScheduleMessageService; +import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.apache.rocketmq.remoting.protocol.body.ConsumerOffsetSerializeWrapper; +import org.apache.rocketmq.remoting.protocol.body.MessageRequestModeSerializeWrapper; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.timer.TimerCheckpoint; +import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.apache.rocketmq.store.timer.TimerMetrics; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.UnsupportedEncodingException; +import java.util.concurrent.ConcurrentHashMap; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class SlaveSynchronizeTest { + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + + private SlaveSynchronize slaveSynchronize; + + @Mock + private BrokerOuterAPI brokerOuterAPI; + + @Mock + private TopicConfigManager topicConfigManager; + + @Mock + private ConsumerOffsetManager consumerOffsetManager; + + @Mock + private MessageStoreConfig messageStoreConfig; + + @Mock + private MessageStore messageStore; + + @Mock + private ScheduleMessageService scheduleMessageService; + + @Mock + private SubscriptionGroupManager subscriptionGroupManager; + + @Mock + private QueryAssignmentProcessor queryAssignmentProcessor; + + @Mock + private MessageRequestModeManager messageRequestModeManager; + + @Mock + private TimerMessageStore timerMessageStore; + + @Mock + private TimerMetrics timerMetrics; + + @Mock + private TimerCheckpoint timerCheckpoint; + + private static final String BROKER_ADDR = "127.0.0.1:10911"; + + @Before + public void init() { + when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService); + when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager); + when(brokerController.getQueryAssignmentProcessor()).thenReturn(queryAssignmentProcessor); + when(brokerController.getMessageStore()).thenReturn(messageStore); + when(brokerController.getTimerMessageStore()).thenReturn(timerMessageStore); + when(brokerController.getTimerCheckpoint()).thenReturn(timerCheckpoint); + when(topicConfigManager.getDataVersion()).thenReturn(new DataVersion()); + when(topicConfigManager.getTopicConfigTable()).thenReturn(new ConcurrentHashMap<>()); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.getOffsetTable()).thenReturn(new ConcurrentHashMap<>()); + when(consumerOffsetManager.getDataVersion()).thenReturn(new DataVersion()); + when(subscriptionGroupManager.getDataVersion()).thenReturn(new DataVersion()); + when(subscriptionGroupManager.getSubscriptionGroupTable()).thenReturn(new ConcurrentHashMap<>()); + when(queryAssignmentProcessor.getMessageRequestModeManager()).thenReturn(messageRequestModeManager); + when(messageRequestModeManager.getMessageRequestModeMap()).thenReturn(new ConcurrentHashMap<>()); + when(messageStoreConfig.isTimerWheelEnable()).thenReturn(true); + when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore); + when(timerMessageStore.isShouldRunningDequeue()).thenReturn(false); + when(timerMessageStore.getTimerMetrics()).thenReturn(timerMetrics); + when(timerMetrics.getDataVersion()).thenReturn(new DataVersion()); + when(timerCheckpoint.getDataVersion()).thenReturn(new DataVersion()); + slaveSynchronize = new SlaveSynchronize(brokerController); + slaveSynchronize.setMasterAddr(BROKER_ADDR); + } + + @Test + public void testSyncAll() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException, UnsupportedEncodingException { + TopicConfig newTopicConfig = new TopicConfig("NewTopic"); + when(brokerOuterAPI.getAllTopicConfig(anyString())).thenReturn(createTopicConfigWrapper(newTopicConfig)); + when(brokerOuterAPI.getAllConsumerOffset(anyString())).thenReturn(createConsumerOffsetWrapper()); + when(brokerOuterAPI.getAllDelayOffset(anyString())).thenReturn(""); + when(brokerOuterAPI.getAllSubscriptionGroupConfig(anyString())).thenReturn(createSubscriptionGroupWrapper()); + when(brokerOuterAPI.getAllMessageRequestMode(anyString())).thenReturn(createMessageRequestModeWrapper()); + when(brokerOuterAPI.getTimerMetrics(anyString())).thenReturn(createTimerMetricsWrapper()); + slaveSynchronize.syncAll(); + Assert.assertEquals(1, this.brokerController.getTopicConfigManager().getDataVersion().getStateVersion()); + Assert.assertEquals(1, this.brokerController.getTopicQueueMappingManager().getDataVersion().getStateVersion()); + Assert.assertEquals(1, consumerOffsetManager.getDataVersion().getStateVersion()); + Assert.assertEquals(1, subscriptionGroupManager.getDataVersion().getStateVersion()); + Assert.assertEquals(1, timerMetrics.getDataVersion().getStateVersion()); + } + + @Test + public void testGetMasterAddr() { + Assert.assertEquals(BROKER_ADDR, slaveSynchronize.getMasterAddr()); + } + + @Test + public void testSyncTimerCheckPoint() throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + when(brokerOuterAPI.getTimerCheckPoint(anyString())).thenReturn(timerCheckpoint); + slaveSynchronize.syncTimerCheckPoint(); + Assert.assertEquals(0, timerCheckpoint.getDataVersion().getStateVersion()); + } + + private TopicConfigAndMappingSerializeWrapper createTopicConfigWrapper(TopicConfig topicConfig) { + TopicConfigAndMappingSerializeWrapper wrapper = new TopicConfigAndMappingSerializeWrapper(); + wrapper.setTopicConfigTable(new ConcurrentHashMap<>()); + wrapper.getTopicConfigTable().put(topicConfig.getTopicName(), topicConfig); + DataVersion dataVersion = new DataVersion(); + dataVersion.setStateVersion(1L); + wrapper.setDataVersion(dataVersion); + wrapper.setMappingDataVersion(dataVersion); + return wrapper; + } + + private ConsumerOffsetSerializeWrapper createConsumerOffsetWrapper() { + ConsumerOffsetSerializeWrapper wrapper = new ConsumerOffsetSerializeWrapper(); + wrapper.setOffsetTable(new ConcurrentHashMap<>()); + DataVersion dataVersion = new DataVersion(); + dataVersion.setStateVersion(1L); + wrapper.setDataVersion(dataVersion); + return wrapper; + } + + private SubscriptionGroupWrapper createSubscriptionGroupWrapper() { + SubscriptionGroupWrapper wrapper = new SubscriptionGroupWrapper(); + wrapper.setSubscriptionGroupTable(new ConcurrentHashMap<>()); + DataVersion dataVersion = new DataVersion(); + dataVersion.setStateVersion(1L); + wrapper.setDataVersion(dataVersion); + return wrapper; + } + + private MessageRequestModeSerializeWrapper createMessageRequestModeWrapper() { + MessageRequestModeSerializeWrapper wrapper = new MessageRequestModeSerializeWrapper(); + wrapper.setMessageRequestModeMap(new ConcurrentHashMap<>()); + return wrapper; + } + + private TimerMetrics.TimerMetricsSerializeWrapper createTimerMetricsWrapper() { + TimerMetrics.TimerMetricsSerializeWrapper wrapper = new TimerMetrics.TimerMetricsSerializeWrapper(); + wrapper.setTimingCount(new ConcurrentHashMap<>()); + DataVersion dataVersion = new DataVersion(); + dataVersion.setStateVersion(1L); + wrapper.setDataVersion(dataVersion); + return wrapper; + } +} From 0e56d8761cefbb3fcb47151ea969e7af375cbb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=88=91=E6=98=AF=E7=AE=A1=E5=B0=8F=E4=BA=AE=5FV0x3f?= <42903364+TeFuirnever@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:26:41 +0800 Subject: [PATCH 071/265] [ISSUE #8417] Add some test cases for org.apache.rocketmq.common.AclConfig (#8418) * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8417 Brief Description add test case for AclConfig in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes [Enhancement] Add test cases for org.apache.rocketmq.common.AclConfig #8417 Fixes #8417 Brief Description add some test cases for org.apache.rocketmq.common.AclConfig. How Did You Test This Change? run test case successfull. --- .../apache/rocketmq/common/AclConfigTest.java | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/AclConfigTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/AclConfigTest.java b/common/src/test/java/org/apache/rocketmq/common/AclConfigTest.java new file mode 100644 index 00000000000..141089f2de0 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/AclConfigTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class AclConfigTest { + + @Test + public void testGetGlobalWhiteAddrsWhenNull() { + AclConfig aclConfig = new AclConfig(); + Assert.assertNull("The globalWhiteAddrs should return null", aclConfig.getGlobalWhiteAddrs()); + } + + @Test + public void testGetGlobalWhiteAddrsWhenEmpty() { + AclConfig aclConfig = new AclConfig(); + List globalWhiteAddrs = new ArrayList<>(); + aclConfig.setGlobalWhiteAddrs(globalWhiteAddrs); + assertNotNull("The globalWhiteAddrs should never return null", aclConfig.getGlobalWhiteAddrs()); + assertEquals("The globalWhiteAddrs list should be empty", 0, aclConfig.getGlobalWhiteAddrs().size()); + } + + @Test + public void testGetGlobalWhiteAddrs() { + AclConfig aclConfig = new AclConfig(); + List expected = Arrays.asList("192.168.1.1", "192.168.1.2"); + aclConfig.setGlobalWhiteAddrs(expected); + assertEquals("Global white addresses should match", expected, aclConfig.getGlobalWhiteAddrs()); + assertEquals("The globalWhiteAddrs list should be equal to 2", 2, aclConfig.getGlobalWhiteAddrs().size()); + } + + @Test + public void testGetPlainAccessConfigsWhenNull() { + AclConfig aclConfig = new AclConfig(); + Assert.assertNull("The plainAccessConfigs should return null", aclConfig.getPlainAccessConfigs()); + } + + @Test + public void testGetPlainAccessConfigsWhenEmpty() { + AclConfig aclConfig = new AclConfig(); + List plainAccessConfigs = new ArrayList<>(); + aclConfig.setPlainAccessConfigs(plainAccessConfigs); + assertNotNull("The plainAccessConfigs should never return null", aclConfig.getPlainAccessConfigs()); + assertEquals("The plainAccessConfigs list should be empty", 0, aclConfig.getPlainAccessConfigs().size()); + } + + @Test + public void testGetPlainAccessConfigs() { + AclConfig aclConfig = new AclConfig(); + List expected = Arrays.asList(new PlainAccessConfig(), new PlainAccessConfig()); + aclConfig.setPlainAccessConfigs(expected); + assertEquals("Plain access configs should match", expected, aclConfig.getPlainAccessConfigs()); + assertEquals("The plainAccessConfigs list should be equal to 2", 2, aclConfig.getPlainAccessConfigs().size()); + } + + @Test + public void testToStringWithNullValues() { + AclConfig aclConfig = new AclConfig(); + String result = aclConfig.toString(); + assertNotNull("toString should not be null", result); + assertEquals("toString should match", "AclConfig{globalWhiteAddrs=null, plainAccessConfigs=null}", result); + } + + @Test + public void testToStringWithEmptyGlobalWhiteAddrsAndPlainAccessConfigs() { + AclConfig aclConfig = new AclConfig(); + aclConfig.setGlobalWhiteAddrs(Collections.emptyList()); + aclConfig.setPlainAccessConfigs(Collections.emptyList()); + String expected = "AclConfig{globalWhiteAddrs=[], plainAccessConfigs=[]}"; + assertEquals(expected, aclConfig.toString()); + } + + @Test + public void testToStringWithNonEmptyGlobalWhiteAddrsAndPlainAccessConfigs() { + AclConfig aclConfig = new AclConfig(); + List globalWhiteAddrs = Collections.singletonList("192.168.1.1"); + aclConfig.setGlobalWhiteAddrs(globalWhiteAddrs); + PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); + List plainAccessConfigs = Collections.singletonList(plainAccessConfig); + aclConfig.setPlainAccessConfigs(plainAccessConfigs); + String expected = "AclConfig{globalWhiteAddrs=[192.168.1.1], plainAccessConfigs=[" + plainAccessConfig + "]}"; + assertEquals("toString should match", expected, aclConfig.toString()); + } +} From 6fb455a1d4dc7416c81ad447fbfe4f9429765609 Mon Sep 17 00:00:00 2001 From: bxfjb <48467309+bxfjb@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:53:39 +0800 Subject: [PATCH 072/265] [ISSUE #8409] Fix tiered storage roll file logic if committing the last part of a file failed (#8410) Co-authored-by: zhaoyuhan --- .../rocketmq/tieredstore/file/FlatAppendFile.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java index d0484137982..b9ba80d08d4 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java @@ -175,8 +175,14 @@ public AppendResult append(ByteBuffer buffer, long timestamp) { FileSegment fileSegment = this.getFileToWrite(); result = fileSegment.append(buffer, timestamp); if (result == AppendResult.FILE_FULL) { - fileSegment.commitAsync().join(); - return this.rollingNewFile(this.getAppendOffset()).append(buffer, timestamp); + boolean commitResult = fileSegment.commitAsync().join(); + log.info("FlatAppendFile#append not successful for the file {} is full, commit result={}", + fileSegment.getPath(), commitResult); + if (commitResult) { + return this.rollingNewFile(this.getAppendOffset()).append(buffer, timestamp); + } else { + return AppendResult.UNKNOWN_ERROR; + } } } finally { fileSegmentLock.writeLock().unlock(); From 6e6319f11a7deb80dc0128ee707f7f9595c9126f Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 24 Jul 2024 14:05:28 +0800 Subject: [PATCH 073/265] [ISSUE #8437] Add more test coverage for ClientRemotingProcessor (#8433) * [ISSUE #8262] Add more test coverage for ClientRemotingProcessor * Update * Update --- .../impl/ClientRemotingProcessorTest.java | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java new file mode 100644 index 00000000000..ed31aa10430 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/ClientRemotingProcessorTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.MQProducerInner; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody; +import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ReplyMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +import static org.apache.rocketmq.common.message.MessageDecoder.NAME_VALUE_SEPARATOR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ClientRemotingProcessorTest { + + @Mock + private MQClientInstance mQClientFactory; + + private ClientRemotingProcessor processor; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultGroup = "defaultGroup"; + + @Before + public void init() throws RemotingException, InterruptedException, MQClientException { + processor = new ClientRemotingProcessor(mQClientFactory); + ClientConfig clientConfig = mock(ClientConfig.class); + when(clientConfig.getNamespace()).thenReturn("namespace"); + when(mQClientFactory.getClientConfig()).thenReturn(clientConfig); + MQProducerInner producerInner = mock(MQProducerInner.class); + when(mQClientFactory.selectProducer(defaultGroup)).thenReturn(producerInner); + } + + @Test + public void testCheckTransactionState() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.CHECK_TRANSACTION_STATE); + when(request.getBody()).thenReturn(getMessageResult()); + CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader(); + when(request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class)).thenReturn(requestHeader); + assertNull(processor.processRequest(ctx, request)); + } + + @Test + public void testNotifyConsumerIdsChanged() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED); + NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader(); + when(request.decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class)).thenReturn(requestHeader); + assertNull(processor.processRequest(ctx, request)); + } + + @Test + public void testResetOffset() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.RESET_CONSUMER_CLIENT_OFFSET); + ResetOffsetBody offsetBody = new ResetOffsetBody(); + when(request.getBody()).thenReturn(RemotingSerializable.encode(offsetBody)); + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + when(request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class)).thenReturn(requestHeader); + assertNull(processor.processRequest(ctx, request)); + } + + @Test + public void testGetConsumeStatus() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT); + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + when(request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class)).thenReturn(requestHeader); + assertNotNull(processor.processRequest(ctx, request)); + } + + @Test + public void testGetConsumerRunningInfo() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.GET_CONSUMER_RUNNING_INFO); + ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo(); + consumerRunningInfo.setJstack("jstack"); + when(mQClientFactory.consumerRunningInfo(anyString())).thenReturn(consumerRunningInfo); + GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader(); + requestHeader.setJstackEnable(true); + requestHeader.setConsumerGroup(defaultGroup); + when(request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class)).thenReturn(requestHeader); + RemotingCommand command = processor.processRequest(ctx, request); + assertNotNull(command); + assertEquals(ResponseCode.SUCCESS, command.getCode()); + } + + @Test + public void testConsumeMessageDirectly() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.CONSUME_MESSAGE_DIRECTLY); + when(request.getBody()).thenReturn(getMessageResult()); + ConsumeMessageDirectlyResult directlyResult = mock(ConsumeMessageDirectlyResult.class); + when(mQClientFactory.consumeMessageDirectly(any(MessageExt.class), anyString(), anyString())).thenReturn(directlyResult); + ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader(); + requestHeader.setConsumerGroup(defaultGroup); + requestHeader.setBrokerName(defaultBroker); + when(request.decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class)).thenReturn(requestHeader); + RemotingCommand command = processor.processRequest(ctx, request); + assertNotNull(command); + assertEquals(ResponseCode.SUCCESS, command.getCode()); + } + + @Test + public void testReceiveReplyMessage() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand request = mock(RemotingCommand.class); + when(request.getCode()).thenReturn(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT); + when(request.getBody()).thenReturn(getMessageResult()); + when(request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class)).thenReturn(createReplyMessageRequestHeader()); + when(request.getBody()).thenReturn(new byte[1]); + RemotingCommand command = processor.processRequest(ctx, request); + assertNotNull(command); + assertEquals(ResponseCode.SUCCESS, command.getCode()); + } + + private ReplyMessageRequestHeader createReplyMessageRequestHeader() { + ReplyMessageRequestHeader result = new ReplyMessageRequestHeader(); + result.setTopic(defaultTopic); + result.setQueueId(0); + result.setStoreTimestamp(System.currentTimeMillis()); + result.setBornTimestamp(System.currentTimeMillis()); + result.setReconsumeTimes(1); + result.setBornHost("127.0.0.1:12911"); + result.setStoreHost("127.0.0.1:10911"); + result.setSysFlag(1); + result.setFlag(1); + result.setProperties("CORRELATION_ID" + NAME_VALUE_SEPARATOR + "1"); + return result; + } + + private byte[] getMessageResult() throws Exception { + byte[] bytes = MessageDecoder.encode(createMessageExt(), false); + ByteBuffer byteBuffer = ByteBuffer.allocate(bytes.length); + byteBuffer.put(bytes); + return byteBuffer.array(); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } +} From c080e6bbf208bc91f06ba97beb37241d7d0c20a0 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 25 Jul 2024 11:38:31 +0800 Subject: [PATCH 074/265] [ISSUE #8438] Fix broker return two messages when query message and index service bug (#8439) --- .../org/apache/rocketmq/tieredstore/TieredMessageStore.java | 3 +++ .../org/apache/rocketmq/tieredstore/file/FlatAppendFile.java | 1 + .../org/apache/rocketmq/tieredstore/index/IndexStoreFile.java | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index 99d586ae236..9a25f85a6b8 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -460,6 +460,9 @@ public synchronized void shutdown() { if (flatFileStore != null) { flatFileStore.shutdown(); } + if (indexService != null) { + indexService.shutdown(); + } if (storeExecutor != null) { storeExecutor.shutdown(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java index b9ba80d08d4..0c20a1cfb4f 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java @@ -90,6 +90,7 @@ public void recoverFileSize() { public void initOffset(long offset) { if (this.fileSegmentTable.isEmpty()) { FileSegment fileSegment = fileSegmentFactory.createSegment(fileType, filePath, offset); + fileSegment.initPosition(fileSegment.getSize()); this.flushFileSegmentMeta(fileSegment); this.fileSegmentTable.add(fileSegment); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java index 180399332e4..f9604b43e6f 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java @@ -287,7 +287,9 @@ protected CompletableFuture> queryAsyncFromUnsealedFile( buffer.position(this.getItemPosition(slotValue)); buffer.get(bytes); IndexItem indexItem = new IndexItem(bytes); - if (hashCode == indexItem.getHashCode()) { + long storeTimestamp = indexItem.getTimeDiff() + beginTimestamp.get(); + if (hashCode == indexItem.getHashCode() && + beginTime <= storeTimestamp && storeTimestamp <= endTime) { result.add(indexItem); if (result.size() > maxCount) { break; From 59c8609f4338db1e20a802c99f21a3a6dca55895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=88=91=E6=98=AF=E7=AE=A1=E5=B0=8F=E4=BA=AE=5FV0x3f?= <42903364+TeFuirnever@users.noreply.github.com> Date: Thu, 25 Jul 2024 19:03:14 +0800 Subject: [PATCH 075/265] [ISSUE #8434] Add test cases for org.apache.rocketmq.common.action (#8435) * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8417 Brief Description add test case for AclConfig in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * [ISSUE apache#8434] Add more test coverage for Add test cases for org.apache.rocketmq.common.action --- .../rocketmq/common/action/ActionTest.java | 117 ++++++++++++++++++ .../common/action/RocketMQActionTest.java | 50 ++++++++ 2 files changed, 167 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java b/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java new file mode 100644 index 00000000000..e3c1b9a3fcb --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/action/ActionTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.action; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ActionTest { + + @Test + public void getByName_NullName_ReturnsNull() { + Action result = Action.getByName("null"); + assertNull(result); + } + + @Test + public void getByName_UnknownName_ReturnsUnknown() { + Action result = Action.getByName("unknown"); + assertEquals(Action.UNKNOWN, result); + } + + @Test + public void getByName_AllName_ReturnsAll() { + Action result = Action.getByName("All"); + assertEquals(Action.ALL, result); + } + + @Test + public void getByName_AnyName_ReturnsAny() { + Action result = Action.getByName("Any"); + assertEquals(Action.ANY, result); + } + + @Test + public void getByName_PubName_ReturnsPub() { + Action result = Action.getByName("Pub"); + assertEquals(Action.PUB, result); + } + + @Test + public void getByName_SubName_ReturnsSub() { + Action result = Action.getByName("Sub"); + assertEquals(Action.SUB, result); + } + + @Test + public void getByName_CreateName_ReturnsCreate() { + Action result = Action.getByName("Create"); + assertEquals(Action.CREATE, result); + } + + @Test + public void getByName_UpdateName_ReturnsUpdate() { + Action result = Action.getByName("Update"); + assertEquals(Action.UPDATE, result); + } + + @Test + public void getByName_DeleteName_ReturnsDelete() { + Action result = Action.getByName("Delete"); + assertEquals(Action.DELETE, result); + } + + @Test + public void getByName_GetName_ReturnsGet() { + Action result = Action.getByName("Get"); + assertEquals(Action.GET, result); + } + + @Test + public void getByName_ListName_ReturnsList() { + Action result = Action.getByName("List"); + assertEquals(Action.LIST, result); + } + + @Test + public void getCode_ReturnsCorrectCode() { + assertEquals((byte) 1, Action.ALL.getCode()); + assertEquals((byte) 2, Action.ANY.getCode()); + assertEquals((byte) 3, Action.PUB.getCode()); + assertEquals((byte) 4, Action.SUB.getCode()); + assertEquals((byte) 5, Action.CREATE.getCode()); + assertEquals((byte) 6, Action.UPDATE.getCode()); + assertEquals((byte) 7, Action.DELETE.getCode()); + assertEquals((byte) 8, Action.GET.getCode()); + assertEquals((byte) 9, Action.LIST.getCode()); + } + + @Test + public void getName_ReturnsCorrectName() { + assertEquals("All", Action.ALL.getName()); + assertEquals("Any", Action.ANY.getName()); + assertEquals("Pub", Action.PUB.getName()); + assertEquals("Sub", Action.SUB.getName()); + assertEquals("Create", Action.CREATE.getName()); + assertEquals("Update", Action.UPDATE.getName()); + assertEquals("Delete", Action.DELETE.getName()); + assertEquals("Get", Action.GET.getName()); + assertEquals("List", Action.LIST.getName()); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java b/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java new file mode 100644 index 00000000000..373a1f620c4 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/action/RocketMQActionTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.action; + +import org.apache.rocketmq.common.resource.ResourceType; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class RocketMQActionTest { + + @Test + public void testRocketMQAction_DefaultResourceType_CustomisedValueAndActionArray() { + RocketMQAction annotation = DemoClass.class.getAnnotation(RocketMQAction.class); + assertEquals(0, annotation.value()); + assertEquals(ResourceType.UNKNOWN, annotation.resource()); + assertArrayEquals(new Action[] {}, annotation.action()); + } + + @Test + public void testRocketMQAction_CustomisedValueAndResourceTypeAndActionArray() { + RocketMQAction annotation = CustomisedDemoClass.class.getAnnotation(RocketMQAction.class); + assertEquals(1, annotation.value()); + assertEquals(ResourceType.TOPIC, annotation.resource()); + assertArrayEquals(new Action[] {Action.CREATE, Action.DELETE}, annotation.action()); + } + + @RocketMQAction(value = 0, resource = ResourceType.UNKNOWN, action = {}) + private static class DemoClass { + } + + @RocketMQAction(value = 1, resource = ResourceType.TOPIC, action = {Action.CREATE, Action.DELETE}) + private static class CustomisedDemoClass { + } +} From 304987d341f17f00e88a4ef2396f04950b4e1720 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 26 Jul 2024 10:23:55 +0800 Subject: [PATCH 076/265] [ISSUE #8446] Add more test coverage for MQClientInstance (#8447) * [ISSUE #8446] Add more test coverage for MQClientInstance * Update * Update --- .../impl/factory/MQClientInstanceTest.java | 402 ++++++++++++++++-- 1 file changed, 361 insertions(+), 41 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java index acd792b8625..d71bc25b9b3 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/factory/MQClientInstanceTest.java @@ -16,77 +16,116 @@ */ package org.apache.rocketmq.client.impl.factory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.admin.MQAdminExtInner; +import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.ConsumeMessageService; +import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; import org.apache.rocketmq.client.impl.consumer.MQConsumerInner; +import org.apache.rocketmq.client.impl.consumer.ProcessQueue; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.MessageQueueAssignment; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.common.HeartbeatV2Result; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.route.QueueData; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class MQClientInstanceTest { - private MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); - private String topic = "FooBar"; - private String group = "FooBarGroup"; - private ConcurrentMap> brokerAddrTable = new ConcurrentHashMap<>(); - @Before - public void init() throws Exception { - FieldUtils.writeDeclaredField(mqClientInstance, "brokerAddrTable", brokerAddrTable, true); - } + @Mock + private MQClientAPIImpl mQClientAPIImpl; - @Test - public void testTopicRouteData2TopicPublishInfo() { - TopicRouteData topicRouteData = new TopicRouteData(); + @Mock + private RemotingClient remotingClient; - topicRouteData.setFilterServerTable(new HashMap<>()); - List brokerDataList = new ArrayList<>(); - BrokerData brokerData = new BrokerData(); - brokerData.setBrokerName("BrokerA"); - brokerData.setCluster("DefaultCluster"); - HashMap brokerAddrs = new HashMap<>(); - brokerAddrs.put(0L, "127.0.0.1:10911"); - brokerData.setBrokerAddrs(brokerAddrs); - brokerDataList.add(brokerData); - topicRouteData.setBrokerDatas(brokerDataList); + @Mock + private ClientConfig clientConfig; - List queueDataList = new ArrayList<>(); - QueueData queueData = new QueueData(); - queueData.setBrokerName("BrokerA"); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSysFlag(0); - queueDataList.add(queueData); - topicRouteData.setQueueDatas(queueDataList); + private final MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + + private final String topic = "FooBar"; + + private final String group = "FooBarGroup"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final String defaultBroker = "BrokerA"; + + private final ConcurrentMap> brokerAddrTable = new ConcurrentHashMap<>(); - TopicPublishInfo topicPublishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData); + private final ConcurrentMap consumerTable = new ConcurrentHashMap<>(); - assertThat(topicPublishInfo.isHaveTopicRouterInfo()).isFalse(); - assertThat(topicPublishInfo.getMessageQueueList().size()).isEqualTo(4); + private final ConcurrentMap topicRouteTable = new ConcurrentHashMap<>(); + + @Before + public void init() throws Exception { + when(mQClientAPIImpl.getRemotingClient()).thenReturn(remotingClient); + FieldUtils.writeDeclaredField(mqClientInstance, "brokerAddrTable", brokerAddrTable, true); + FieldUtils.writeDeclaredField(mqClientInstance, "mQClientAPIImpl", mQClientAPIImpl, true); + FieldUtils.writeDeclaredField(mqClientInstance, "consumerTable", consumerTable, true); + FieldUtils.writeDeclaredField(mqClientInstance, "clientConfig", clientConfig, true); + FieldUtils.writeDeclaredField(mqClientInstance, "topicRouteTable", topicRouteTable, true); } @Test @@ -131,7 +170,7 @@ public void testRegisterProducer() { } @Test - public void testRegisterConsumer() throws RemotingException, InterruptedException, MQBrokerException { + public void testRegisterConsumer() { boolean flag = mqClientInstance.registerConsumer(group, mock(MQConsumerInner.class)); assertThat(flag).isTrue(); @@ -143,7 +182,6 @@ public void testRegisterConsumer() throws RemotingException, InterruptedExceptio assertThat(flag).isTrue(); } - @Test public void testConsumerRunningInfoWhenConsumersIsEmptyOrNot() throws RemotingException, InterruptedException, MQBrokerException { MQConsumerInner mockConsumerInner = mock(MQConsumerInner.class); @@ -181,4 +219,286 @@ public void testRegisterAdminExt() { assertThat(flag).isTrue(); } + @Test + public void testTopicRouteData2TopicPublishInfo() { + TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, createTopicRouteData()); + assertThat(actual.isHaveTopicRouterInfo()).isFalse(); + assertThat(actual.getMessageQueueList().size()).isEqualTo(4); + } + + @Test + public void testTopicRouteData2TopicPublishInfoWithOrderTopicConf() { + TopicRouteData topicRouteData = createTopicRouteData(); + when(topicRouteData.getOrderTopicConf()).thenReturn("127.0.0.1:4"); + TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData); + assertFalse(actual.isHaveTopicRouterInfo()); + assertEquals(4, actual.getMessageQueueList().size()); + } + + @Test + public void testTopicRouteData2TopicPublishInfoWithTopicQueueMappingByBroker() { + TopicRouteData topicRouteData = createTopicRouteData(); + when(topicRouteData.getTopicQueueMappingByBroker()).thenReturn(Collections.singletonMap(topic, new TopicQueueMappingInfo())); + TopicPublishInfo actual = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData); + assertFalse(actual.isHaveTopicRouterInfo()); + assertEquals(0, actual.getMessageQueueList().size()); + } + + @Test + public void testTopicRouteData2TopicSubscribeInfo() { + TopicRouteData topicRouteData = createTopicRouteData(); + when(topicRouteData.getTopicQueueMappingByBroker()).thenReturn(Collections.singletonMap(topic, new TopicQueueMappingInfo())); + Set actual = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testParseOffsetTableFromBroker() { + Map offsetTable = new HashMap<>(); + offsetTable.put(new MessageQueue(), 0L); + Map actual = mqClientInstance.parseOffsetTableFromBroker(offsetTable, "defaultNamespace"); + assertNotNull(actual); + assertEquals(1, actual.size()); + } + + @Test + public void testCheckClientInBroker() throws MQClientException, RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, InterruptedException { + doThrow(new MQClientException("checkClientInBroker exception", null)).when(mQClientAPIImpl).checkClientInBroker( + any(), + any(), + any(), + any(SubscriptionData.class), + anyLong()); + topicRouteTable.put(topic, createTopicRouteData()); + MQConsumerInner mqConsumerInner = createMQConsumerInner(); + mqConsumerInner.subscriptions().clear(); + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTopic(topic); + subscriptionData.setExpressionType("type"); + mqConsumerInner.subscriptions().add(subscriptionData); + consumerTable.put(group, mqConsumerInner); + Throwable thrown = assertThrows(MQClientException.class, mqClientInstance::checkClientInBroker); + assertTrue(thrown.getMessage().contains("checkClientInBroker exception")); + } + + @Test + public void testSendHeartbeatToBrokerV1() { + consumerTable.put(group, createMQConsumerInner()); + assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr)); + } + + @Test + public void testSendHeartbeatToBrokerV2() throws MQBrokerException, RemotingException, InterruptedException { + consumerTable.put(group, createMQConsumerInner()); + when(clientConfig.isUseHeartbeatV2()).thenReturn(true); + HeartbeatV2Result heartbeatV2Result = mock(HeartbeatV2Result.class); + when(heartbeatV2Result.isSupportV2()).thenReturn(true); + when(mQClientAPIImpl.sendHeartbeatV2(any(), any(HeartbeatData.class), anyLong())).thenReturn(heartbeatV2Result); + assertTrue(mqClientInstance.sendHeartbeatToBroker(0L, defaultBroker, defaultBrokerAddr)); + } + + @Test + public void testSendHeartbeatToAllBrokerWithLockV1() { + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock()); + } + + @Test + public void testSendHeartbeatToAllBrokerWithLockV2() { + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + when(clientConfig.isUseHeartbeatV2()).thenReturn(true); + assertTrue(mqClientInstance.sendHeartbeatToAllBrokerWithLock()); + } + + @Test + public void testUpdateTopicRouteInfoFromNameServer() throws RemotingException, InterruptedException, MQClientException { + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class); + TopicRouteData topicRouteData = createTopicRouteData(); + when(mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(anyLong())).thenReturn(topicRouteData); + assertFalse(mqClientInstance.updateTopicRouteInfoFromNameServer(topic, true, defaultMQProducer)); + } + + @Test + public void testFindBrokerAddressInAdmin() { + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + FindBrokerResult actual = mqClientInstance.findBrokerAddressInAdmin(defaultBroker); + assertNotNull(actual); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + } + + @Test + public void testFindBrokerAddressInSubscribeWithOneBroker() throws IllegalAccessException { + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + ConcurrentMap> brokerVersionTable = new ConcurrentHashMap<>(); + HashMap addressMap = new HashMap<>(); + addressMap.put(defaultBrokerAddr, 0); + brokerVersionTable.put(defaultBroker, addressMap); + FieldUtils.writeDeclaredField(mqClientInstance, "brokerVersionTable", brokerVersionTable, true); + FindBrokerResult actual = mqClientInstance.findBrokerAddressInSubscribe(defaultBroker, 1L, false); + assertNotNull(actual); + assertEquals(defaultBrokerAddr, actual.getBrokerAddr()); + } + + @Test + public void testFindConsumerIdList() { + topicRouteTable.put(topic, createTopicRouteData()); + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + List actual = mqClientInstance.findConsumerIdList(topic, group); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testQueryAssignment() throws MQBrokerException, RemotingException, InterruptedException { + topicRouteTable.put(topic, createTopicRouteData()); + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + Set actual = mqClientInstance.queryAssignment(topic, group, "", MessageModel.CLUSTERING, 1000); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testResetOffset() throws IllegalAccessException { + topicRouteTable.put(topic, createTopicRouteData()); + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + Map offsetTable = new HashMap<>(); + offsetTable.put(createMessageQueue(), 0L); + mqClientInstance.resetOffset(topic, group, offsetTable); + Field consumerTableField = FieldUtils.getDeclaredField(mqClientInstance.getClass(), "consumerTable", true); + ConcurrentMap consumerTable = (ConcurrentMap) consumerTableField.get(mqClientInstance); + DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) consumerTable.get(group); + verify(consumer).suspend(); + verify(consumer).resume(); + verify(consumer, times(1)) + .updateConsumeOffset( + any(MessageQueue.class), + eq(0L)); + } + + @Test + public void testGetConsumerStatus() { + topicRouteTable.put(topic, createTopicRouteData()); + brokerAddrTable.put(defaultBroker, createBrokerAddrMap()); + consumerTable.put(group, createMQConsumerInner()); + Map actual = mqClientInstance.getConsumerStatus(topic, group); + assertNotNull(actual); + assertEquals(0, actual.size()); + } + + @Test + public void testGetAnExistTopicRouteData() { + topicRouteTable.put(topic, createTopicRouteData()); + TopicRouteData actual = mqClientInstance.getAnExistTopicRouteData(topic); + assertNotNull(actual); + assertNotNull(actual.getQueueDatas()); + assertNotNull(actual.getBrokerDatas()); + } + + @Test + public void testConsumeMessageDirectly() { + consumerTable.put(group, createMQConsumerInner()); + assertNull(mqClientInstance.consumeMessageDirectly(createMessageExt(), group, defaultBroker)); + } + + @Test + public void testQueryTopicRouteData() { + consumerTable.put(group, createMQConsumerInner()); + topicRouteTable.put(topic, createTopicRouteData()); + TopicRouteData actual = mqClientInstance.queryTopicRouteData(topic); + assertNotNull(actual); + assertNotNull(actual.getQueueDatas()); + assertNotNull(actual.getBrokerDatas()); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(topic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, group); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + long curTime = System.currentTimeMillis(); + result.setBornTimestamp(curTime - 1000); + result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime); + result.setKeys("keys"); + result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG); + result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } + + private MessageQueue createMessageQueue() { + MessageQueue result = new MessageQueue(); + result.setQueueId(0); + result.setBrokerName(defaultBroker); + result.setTopic(topic); + return result; + } + + private TopicRouteData createTopicRouteData() { + TopicRouteData result = mock(TopicRouteData.class); + when(result.getBrokerDatas()).thenReturn(createBrokerDatas()); + when(result.getQueueDatas()).thenReturn(createQueueDatas()); + return result; + } + + private HashMap createBrokerAddrMap() { + HashMap result = new HashMap<>(); + result.put(0L, defaultBrokerAddr); + return result; + } + + private MQConsumerInner createMQConsumerInner() { + DefaultMQPushConsumerImpl result = mock(DefaultMQPushConsumerImpl.class); + Set subscriptionDataSet = new HashSet<>(); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + subscriptionDataSet.add(subscriptionData); + when(result.subscriptions()).thenReturn(subscriptionDataSet); + RebalanceImpl rebalanceImpl = mock(RebalanceImpl.class); + ConcurrentMap processQueueMap = new ConcurrentHashMap<>(); + ProcessQueue processQueue = new ProcessQueue(); + processQueueMap.put(createMessageQueue(), processQueue); + when(rebalanceImpl.getProcessQueueTable()).thenReturn(processQueueMap); + when(result.getRebalanceImpl()).thenReturn(rebalanceImpl); + OffsetStore offsetStore = mock(OffsetStore.class); + when(result.getOffsetStore()).thenReturn(offsetStore); + ConsumeMessageService consumeMessageService = mock(ConsumeMessageService.class); + when(result.getConsumeMessageService()).thenReturn(consumeMessageService); + return result; + } + + private List createQueueDatas() { + QueueData queueData = new QueueData(); + queueData.setBrokerName(defaultBroker); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + return Collections.singletonList(queueData); + } + + private List createBrokerDatas() { + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName(defaultBroker); + String defaultCluster = "defaultCluster"; + brokerData.setCluster(defaultCluster); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, defaultBrokerAddr); + brokerData.setBrokerAddrs(brokerAddrs); + return Collections.singletonList(brokerData); + } } From 3722431c2593a5fc568d415d860001c690c5a5ad Mon Sep 17 00:00:00 2001 From: yx9o Date: Sun, 28 Jul 2024 17:11:15 +0800 Subject: [PATCH 077/265] [ISSUE #8458] Add more test coverage for ProcessQueue (#8459) * [ISSUE #8458] Add more test coverage for ProcessQueue * Update * Update * Update --- .../impl/consumer/ProcessQueueTest.java | 82 +++++++++++++++++-- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java index be0bd29f79f..a8afd4a233a 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ProcessQueueTest.java @@ -16,17 +16,32 @@ */ package org.apache.rocketmq.client.impl.consumer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo; import org.assertj.core.util.Lists; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; + import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) public class ProcessQueueTest { @@ -78,7 +93,7 @@ public void testContainsMessage() { } @Test - public void testFillProcessQueueInfo() { + public void testFillProcessQueueInfo() throws IllegalAccessException { ProcessQueue pq = new ProcessQueue(); pq.putMessage(createMessageList(102400)); @@ -101,6 +116,57 @@ public void testFillProcessQueueInfo() { pq.commit(); pq.fillProcessQueueInfo(processQueueInfo); assertThat(processQueueInfo.getCachedMsgSizeInMiB()).isEqualTo(0); + + TreeMap consumingMsgOrderlyTreeMap = new TreeMap<>(); + consumingMsgOrderlyTreeMap.put(0L, createMessageList(1).get(0)); + FieldUtils.writeDeclaredField(pq, "consumingMsgOrderlyTreeMap", consumingMsgOrderlyTreeMap, true); + pq.fillProcessQueueInfo(processQueueInfo); + assertEquals(0, processQueueInfo.getTransactionMsgMinOffset()); + assertEquals(0, processQueueInfo.getTransactionMsgMaxOffset()); + assertEquals(1, processQueueInfo.getTransactionMsgCount()); + } + + @Test + public void testPopRequest() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + ProcessQueue processQueue = createProcessQueue(); + MessageExt messageExt = createMessageList(1).get(0); + messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() - 20 * 60 * 1000L + ""); + processQueue.getMsgTreeMap().put(0L, messageExt); + DefaultMQPushConsumer pushConsumer = mock(DefaultMQPushConsumer.class); + processQueue.cleanExpiredMsg(pushConsumer); + verify(pushConsumer).sendMessageBack(any(MessageExt.class), eq(3)); + } + + @Test + public void testRollback() throws IllegalAccessException { + ProcessQueue processQueue = createProcessQueue(); + processQueue.rollback(); + Field consumingMsgOrderlyTreeMapField = FieldUtils.getDeclaredField(processQueue.getClass(), "consumingMsgOrderlyTreeMap", true); + TreeMap consumingMsgOrderlyTreeMap = (TreeMap) consumingMsgOrderlyTreeMapField.get(processQueue); + assertEquals(0, consumingMsgOrderlyTreeMap.size()); + } + + @Test + public void testHasTempMessage() { + ProcessQueue processQueue = createProcessQueue(); + assertFalse(processQueue.hasTempMessage()); + } + + @Test + public void testProcessQueue() { + ProcessQueue processQueue1 = createProcessQueue(); + ProcessQueue processQueue2 = createProcessQueue(); + assertEquals(processQueue1.getMsgAccCnt(), processQueue2.getMsgAccCnt()); + assertEquals(processQueue1.getTryUnlockTimes(), processQueue2.getTryUnlockTimes()); + assertEquals(processQueue1.getLastLockTimestamp(), processQueue2.getLastLockTimestamp()); + assertEquals(processQueue1.getLastPullTimestamp(), processQueue2.getLastPullTimestamp()); + } + + private ProcessQueue createProcessQueue() { + ProcessQueue result = new ProcessQueue(); + result.setMsgAccCnt(1); + result.incTryUnlockTimes(); + return result; } private List createMessageList() { @@ -108,13 +174,15 @@ private List createMessageList() { } private List createMessageList(int count) { - List messageExtList = new ArrayList<>(); + List result = new ArrayList<>(); for (int i = 0; i < count; i++) { MessageExt messageExt = new MessageExt(); messageExt.setQueueOffset(i); messageExt.setBody(new byte[123]); - messageExtList.add(messageExt); + messageExt.setKeys("keys" + i); + messageExt.getProperties().put(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, System.currentTimeMillis() + ""); + result.add(messageExt); } - return messageExtList; + return result; } } From cc484d56805b8387f0660809ae7e3117a8fd1c46 Mon Sep 17 00:00:00 2001 From: dinglei Date: Sun, 28 Jul 2024 17:12:59 +0800 Subject: [PATCH 078/265] [ISSUE #8454] Active brokers number should be initailized to 1 in broker heartbeat manager. (#8453) * active brokers should be 1 on computing if absent * active brokers number should be initailized to 1 in broker heartbeat manager. --- .../impl/heartbeat/DefaultBrokerHeartbeatManager.java | 2 +- .../controller/impl/heartbeat/RaftBrokerHeartBeatManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java index 5ec298a383e..05d742fb7b0 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java @@ -184,7 +184,7 @@ public Map> getActiveBrokersNum() { .forEach(id -> { map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>()); map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) -> - num == null ? 0 : num + 1 + num == null ? 1 : num + 1 ); }); return map; diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java index 99f7b34d4a4..d981ff430c9 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/RaftBrokerHeartBeatManager.java @@ -263,7 +263,7 @@ public Map> getActiveBrokersNum() { .forEach(id -> { map.computeIfAbsent(id.getClusterName(), k -> new HashMap<>()); map.get(id.getClusterName()).compute(id.getBrokerName(), (broker, num) -> - num == null ? 0 : num + 1 + num == null ? 1 : num + 1 ); }); return map; From 263f0fbdd6e972ec2de117a476ac404aa9d6c591 Mon Sep 17 00:00:00 2001 From: Qiu <472594921@qq.com> Date: Mon, 29 Jul 2024 13:46:46 +0800 Subject: [PATCH 079/265] [ISSUE #8448] commitlog class comment optimize --- .../java/org/apache/rocketmq/store/CommitLog.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 1dd60523a58..f707d8fbd87 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -1964,23 +1964,28 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); } - int pos = 4 + 4 + 4 + 4 + 4; + int pos = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4; // 5 FLAG // 6 QUEUEOFFSET preEncodeBuffer.putLong(pos, queueOffset); pos += 8; // 7 PHYSICALOFFSET preEncodeBuffer.putLong(pos, fileFromOffset + byteBuffer.position()); + pos += 8; int ipLen = (msgInner.getSysFlag() & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 4 + 4 : 16 + 4; - // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST, 11 STORETIMESTAMP - pos += 8 + 4 + 8 + ipLen; - // refresh store time stamp in lock + // 8 SYSFLAG, 9 BORNTIMESTAMP, 10 BORNHOST + pos += 4 + 8 + ipLen; + // 11 STORETIMESTAMP refresh store time stamp in lock preEncodeBuffer.putLong(pos, msgInner.getStoreTimestamp()); if (enabledAppendPropCRC) { // 18 CRC32 int checkSize = msgLen - crc32ReservedLength; ByteBuffer tmpBuffer = preEncodeBuffer.duplicate(); tmpBuffer.limit(tmpBuffer.position() + checkSize); - int crc32 = UtilAll.crc32(tmpBuffer); + int crc32 = UtilAll.crc32(tmpBuffer); // UtilAll.crc32 function will change the position to limit of the buffer tmpBuffer.limit(tmpBuffer.position() + crc32ReservedLength); MessageDecoder.createCrc32(tmpBuffer, crc32); } From 217fc8da53ca5f49e164f772112469cb1359e73c Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:11:16 +0800 Subject: [PATCH 080/265] [ISSUE #8429] Fix trace message loss when traffic is heavy (#8430) --- .../apache/rocketmq/client/ClientConfig.java | 11 + .../consumer/DefaultLitePullConsumer.java | 14 +- .../consumer/DefaultMQPushConsumer.java | 97 +++--- .../client/producer/DefaultMQProducer.java | 2 +- .../client/trace/AsyncTraceDispatcher.java | 292 +++++++----------- .../client/trace/TraceDataEncoder.java | 5 +- 6 files changed, 187 insertions(+), 234 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index 0fc04fcccb2..696b073b373 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageQueue; @@ -64,6 +65,8 @@ public class ClientConfig { */ private int persistConsumerOffsetInterval = 1000 * 5; private long pullTimeDelayMillsWhenException = 1000; + + private int traceMsgBatchNum = 10; private boolean unitMode = false; private String unitName; private boolean decodeReadBody = Boolean.parseBoolean(System.getProperty(DECODE_READ_BODY, "true")); @@ -127,6 +130,14 @@ public String buildMQClientId() { return sb.toString(); } + public int getTraceMsgBatchNum() { + return traceMsgBatchNum; + } + + public void setTraceMsgBatchNum(int traceMsgBatchNum) { + this.traceMsgBatchNum = traceMsgBatchNum; + } + public String getClientIP() { return clientIP; } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 3364df48f89..20857f14e08 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -200,7 +200,7 @@ public DefaultLitePullConsumer(RPCHook rpcHook) { * Constructor specifying consumer group, RPC hook * * @param consumerGroup Consumer group. - * @param rpcHook RPC hook to execute before each remoting command. + * @param rpcHook RPC hook to execute before each remoting command. */ public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) { this.consumerGroup = consumerGroup; @@ -213,7 +213,7 @@ public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) { * Constructor specifying namespace, consumer group and RPC hook. * * @param consumerGroup Consumer group. - * @param rpcHook RPC hook to execute before each remoting command. + * @param rpcHook RPC hook to execute before each remoting command. */ @Deprecated public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) { @@ -270,6 +270,7 @@ public void subscribe(String topic, MessageSelector messageSelector) throws MQCl public void unsubscribe(String topic) { this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic)); } + @Override public void assign(Collection messageQueues) { defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues)); @@ -338,7 +339,8 @@ public void commit() { this.defaultLitePullConsumerImpl.commitAll(); } - @Override public void commit(Map offsetMap, boolean persist) { + @Override + public void commit(Map offsetMap, boolean persist) { this.defaultLitePullConsumerImpl.commit(offsetMap, persist); } @@ -361,11 +363,11 @@ public Set assignment() throws MQClientException { * @param messageQueueListener */ @Override - public void subscribe(String topic, String subExpression, MessageQueueListener messageQueueListener) throws MQClientException { + public void subscribe(String topic, String subExpression, + MessageQueueListener messageQueueListener) throws MQClientException { this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression, messageQueueListener); } - @Override public void commit(final Set messageQueues, boolean persist) { this.defaultLitePullConsumerImpl.commit(messageQueues, persist); @@ -589,7 +591,7 @@ public TraceDispatcher getTraceDispatcher() { private void setTraceDispatcher() { if (enableTrace) { try { - AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, traceTopic, rpcHook); + AsyncTraceDispatcher traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook); traceDispatcher.getTraceProducer().setUseTLS(this.isUseTLS()); traceDispatcher.setNamespaceV2(namespaceV2); this.traceDispatcher = traceDispatcher; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 38a412c237b..2d9fb73cec4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -51,11 +51,9 @@ /** * In most scenarios, this is the mostly recommended class to consume messages. *

- * * Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on * arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages. *

- * * See quickstart/Consumer in the example module for a typical usage. *

* @@ -76,7 +74,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve * load balance. It's required and needs to be globally unique. *

- * * See here for further discussion. */ private String consumerGroup; @@ -84,13 +81,11 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume /** * Message model defines the way how messages are delivered to each consumer clients. *

- * * RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with * the same {@link #consumerGroup} would only consume shards of the messages subscribed, which achieves load * balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages * separately. *

- * * This field defaults to clustering. */ private MessageModel messageModel = MessageModel.CLUSTERING; @@ -98,7 +93,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume /** * Consuming point on consumer booting. *

- * * There are three consuming points: *
    *
  • @@ -239,7 +233,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume */ private int pullBatchSize = 32; - private int pullBatchSizeInBytes = 256 * 1024; /** @@ -256,7 +249,6 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume * Max re-consume times. * In concurrently mode, -1 means 16; * In orderly mode, -1 means Integer.MAX_VALUE. - * * If messages are re-consumed more than {@link #maxReconsumeTimes} before success. */ private int maxReconsumeTimes = -1; @@ -312,7 +304,6 @@ public DefaultMQPushConsumer(final String consumerGroup) { this(consumerGroup, null, new AllocateMessageQueueAveragely()); } - /** * Constructor specifying RPC hook. * @@ -326,29 +317,29 @@ public DefaultMQPushConsumer(RPCHook rpcHook) { * Constructor specifying consumer group, RPC hook. * * @param consumerGroup Consumer group. - * @param rpcHook RPC hook to execute before each remoting command. + * @param rpcHook RPC hook to execute before each remoting command. */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook) { this(consumerGroup, rpcHook, new AllocateMessageQueueAveragely()); } - /** * Constructor specifying consumer group, enabled msg trace flag and customized trace topic name. * - * @param consumerGroup Consumer group. - * @param enableMsgTrace Switch flag instance for message trace. + * @param consumerGroup Consumer group. + * @param enableMsgTrace Switch flag instance for message trace. * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. */ - public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, final String customizedTraceTopic) { + public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, + final String customizedTraceTopic) { this(consumerGroup, null, new AllocateMessageQueueAveragely(), enableMsgTrace, customizedTraceTopic); } /** * Constructor specifying consumer group, RPC hook and message queue allocating algorithm. * - * @param consumerGroup Consume queue. - * @param rpcHook RPC hook to execute before each remoting command. + * @param consumerGroup Consume queue. + * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy Message queue allocating algorithm. */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, @@ -359,14 +350,15 @@ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, /** * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name. * - * @param consumerGroup Consume queue. - * @param rpcHook RPC hook to execute before each remoting command. + * @param consumerGroup Consume queue. + * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy message queue allocating algorithm. - * @param enableMsgTrace Switch flag instance for message trace. - * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. */ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, - AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) { + AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, + final String customizedTraceTopic) { this.consumerGroup = consumerGroup; this.rpcHook = rpcHook; this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; @@ -378,7 +370,7 @@ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, /** * Constructor specifying namespace and consumer group. * - * @param namespace Namespace for this MQ Producer instance. + * @param namespace Namespace for this MQ Producer instance. * @param consumerGroup Consumer group. */ @Deprecated @@ -389,9 +381,9 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup) /** * Constructor specifying namespace, consumer group and RPC hook . * - * @param namespace Namespace for this MQ Producer instance. + * @param namespace Namespace for this MQ Producer instance. * @param consumerGroup Consumer group. - * @param rpcHook RPC hook to execute before each remoting command. + * @param rpcHook RPC hook to execute before each remoting command. */ @Deprecated public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) { @@ -401,9 +393,9 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup, /** * Constructor specifying namespace, consumer group, RPC hook and message queue allocating algorithm. * - * @param namespace Namespace for this MQ Producer instance. - * @param consumerGroup Consume queue. - * @param rpcHook RPC hook to execute before each remoting command. + * @param namespace Namespace for this MQ Producer instance. + * @param consumerGroup Consume queue. + * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy Message queue allocating algorithm. */ @Deprecated @@ -419,16 +411,17 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup, /** * Constructor specifying namespace, consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name. * - * @param namespace Namespace for this MQ Producer instance. - * @param consumerGroup Consume queue. - * @param rpcHook RPC hook to execute before each remoting command. + * @param namespace Namespace for this MQ Producer instance. + * @param consumerGroup Consume queue. + * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy message queue allocating algorithm. - * @param enableMsgTrace Switch flag instance for message trace. - * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. + * @param enableMsgTrace Switch flag instance for message trace. + * @param customizedTraceTopic The name value of message trace topic.If you don't config,you can use the default trace topic name. */ @Deprecated public DefaultMQPushConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook, - AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, final String customizedTraceTopic) { + AllocateMessageQueueStrategy allocateMessageQueueStrategy, boolean enableMsgTrace, + final String customizedTraceTopic) { this.consumerGroup = consumerGroup; this.namespace = namespace; this.rpcHook = rpcHook; @@ -443,7 +436,8 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup, */ @Deprecated @Override - public void createTopic(String key, String newTopic, int queueNum, Map attributes) throws MQClientException { + public void createTopic(String key, String newTopic, int queueNum, + Map attributes) throws MQClientException { createTopic(key, withNamespace(newTopic), queueNum, 0, null); } @@ -457,7 +451,8 @@ public void setUseTLS(boolean useTLS) { */ @Deprecated @Override - public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag, Map attributes) throws MQClientException { + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag, + Map attributes) throws MQClientException { this.defaultMQPushConsumerImpl.createTopic(key, withNamespace(newTopic), queueNum, topicSysFlag); } @@ -677,16 +672,16 @@ public void setSubscription(Map subscription) { /** * Send message back to broker which will be re-delivered in future. - * + *

    * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so * please do not use this method. * - * @param msg Message to send back. + * @param msg Message to send back. * @param delayLevel delay level. - * @throws RemotingException if there is any network-tier error. - * @throws MQBrokerException if there is any broker error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. * @throws InterruptedException if the thread is interrupted. - * @throws MQClientException if there is any client error. + * @throws MQClientException if there is any client error. */ @Deprecated @Override @@ -699,17 +694,17 @@ public void sendMessageBack(MessageExt msg, int delayLevel) /** * Send message back to the broker whose name is brokerName and the message will be re-delivered in * future. - * + *

    * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so * please do not use this method. * - * @param msg Message to send back. + * @param msg Message to send back. * @param delayLevel delay level. * @param brokerName broker name. - * @throws RemotingException if there is any network-tier error. - * @throws MQBrokerException if there is any broker error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. * @throws InterruptedException if the thread is interrupted. - * @throws MQClientException if there is any client error. + * @throws MQClientException if there is any client error. */ @Deprecated @Override @@ -735,7 +730,7 @@ public void start() throws MQClientException { this.defaultMQPushConsumerImpl.start(); if (enableTrace) { try { - AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, traceTopic, rpcHook); + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, getTraceMsgBatchNum(), traceTopic, rpcHook); dispatcher.setHostConsumer(this.defaultMQPushConsumerImpl); dispatcher.setNamespaceV2(namespaceV2); traceDispatcher = dispatcher; @@ -799,9 +794,9 @@ public void registerMessageListener(MessageListenerOrderly messageListener) { /** * Subscribe a topic to consuming subscription. * - * @param topic topic to subscribe. + * @param topic topic to subscribe. * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
    - * if null or * expression,meaning subscribe all + * if null or * expression,meaning subscribe all * @throws MQClientException if there is any client error. */ @Override @@ -812,8 +807,8 @@ public void subscribe(String topic, String subExpression) throws MQClientExcepti /** * Subscribe a topic to consuming subscription. * - * @param topic topic to consume. - * @param fullClassName full class name,must extend org.apache.rocketmq.common.filter. MessageFilter + * @param topic topic to consume. + * @param fullClassName full class name,must extend org.apache.rocketmq.common.filter. MessageFilter * @param filterClassSource class source code,used UTF-8 file encoding,must be responsible for your code safety */ @Override @@ -824,7 +819,7 @@ public void subscribe(String topic, String fullClassName, String filterClassSour /** * Subscribe a topic by message selector. * - * @param topic topic to consume. + * @param topic topic to consume. * @param messageSelector {@link org.apache.rocketmq.client.consumer.MessageSelector} * @see org.apache.rocketmq.client.consumer.MessageSelector#bySql * @see org.apache.rocketmq.client.consumer.MessageSelector#byTag diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 4fd038663b5..3ecd5987c35 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -348,7 +348,7 @@ public void start() throws MQClientException { } if (enableTrace) { try { - AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, traceTopic, rpcHook); + AsyncTraceDispatcher dispatcher = new AsyncTraceDispatcher(producerGroup, TraceDispatcher.Type.PRODUCE, getTraceMsgBatchNum(), traceTopic, rpcHook); dispatcher.setHostProducer(this.defaultMQProducerImpl); dispatcher.setNamespaceV2(this.namespaceV2); traceDispatcher = dispatcher; diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index 1fe19773a5a..6d62617eb8e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -16,19 +16,6 @@ */ package org.apache.rocketmq.client.trace; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.client.AccessChannel; import org.apache.rocketmq.client.common.ThreadLocalIndex; import org.apache.rocketmq.client.exception.MQClientException; @@ -44,68 +31,75 @@ import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.topic.TopicValidator; -import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.RPCHook; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static org.apache.rocketmq.client.trace.TraceConstants.TRACE_INSTANCE_NAME; public class AsyncTraceDispatcher implements TraceDispatcher { - private final static Logger log = LoggerFactory.getLogger(AsyncTraceDispatcher.class); - private final static AtomicInteger COUNTER = new AtomicInteger(); - private final static short MAX_MSG_KEY_SIZE = Short.MAX_VALUE - 10000; - private final int queueSize; - private final int batchSize; + private static final Logger log = LoggerFactory.getLogger(AsyncTraceDispatcher.class); + private static final AtomicInteger COUNTER = new AtomicInteger(); + private static final AtomicInteger INSTANCE_NUM = new AtomicInteger(0); + private volatile boolean stopped = false; + private final int traceInstanceId = INSTANCE_NUM.getAndIncrement(); + private final int batchNum; private final int maxMsgSize; - private final long pollingTimeMil; - private final long waitTimeThresholdMil; private final DefaultMQProducer traceProducer; - private final ThreadPoolExecutor traceExecutor; - // The last discard number of log private AtomicLong discardCount; private Thread worker; + private final ThreadPoolExecutor traceExecutor; private final ArrayBlockingQueue traceContextQueue; - private final HashMap taskQueueByTopic; - private ArrayBlockingQueue appenderQueue; + private final ArrayBlockingQueue appenderQueue; private volatile Thread shutDownHook; - private volatile boolean stopped = false; + private DefaultMQProducerImpl hostProducer; private DefaultMQPushConsumerImpl hostConsumer; private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); - private String dispatcherId = UUID.randomUUID().toString(); private volatile String traceTopicName; private AtomicBoolean isStarted = new AtomicBoolean(false); private volatile AccessChannel accessChannel = AccessChannel.LOCAL; private String group; private Type type; private String namespaceV2; + private final int flushTraceInterval = 5000; + + private long lastFlushTime = System.currentTimeMillis(); - public AsyncTraceDispatcher(String group, Type type, String traceTopicName, RPCHook rpcHook) { - // queueSize is greater than or equal to the n power of 2 of value - this.queueSize = 2048; - this.batchSize = 100; + public AsyncTraceDispatcher(String group, Type type, int batchNum, String traceTopicName, RPCHook rpcHook) { + this.batchNum = Math.min(batchNum, 20);/* max value 20*/ this.maxMsgSize = 128000; - this.pollingTimeMil = 100; - this.waitTimeThresholdMil = 500; this.discardCount = new AtomicLong(0L); - this.traceContextQueue = new ArrayBlockingQueue<>(1024); - this.taskQueueByTopic = new HashMap(); + this.traceContextQueue = new ArrayBlockingQueue<>(2048); this.group = group; this.type = type; - - this.appenderQueue = new ArrayBlockingQueue<>(queueSize); + this.appenderQueue = new ArrayBlockingQueue<>(2048); if (!UtilAll.isBlank(traceTopicName)) { this.traceTopicName = traceTopicName; } else { this.traceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC; } this.traceExecutor = new ThreadPoolExecutor(// - 10, // - 20, // + 2, // + 4, // 1000 * 60, // TimeUnit.MILLISECONDS, // this.appenderQueue, // - new ThreadFactoryImpl("MQTraceSendThread_")); + new ThreadFactoryImpl("MQTraceSendThread_" + traceInstanceId + "_")); traceProducer = getAndCreateTraceProducer(rpcHook); } @@ -153,7 +147,6 @@ public void setNamespaceV2(String namespaceV2) { this.namespaceV2 = namespaceV2; } - @Override public void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClientException { if (isStarted.compareAndSet(false, true)) { traceProducer.setNamesrvAddr(nameSrvAddr); @@ -163,7 +156,8 @@ public void start(String nameSrvAddr, AccessChannel accessChannel) throws MQClie traceProducer.start(); } this.accessChannel = accessChannel; - this.worker = new Thread(new AsyncRunnable(), "MQ-AsyncTraceDispatcher-Thread-" + dispatcherId); + this.worker = new ThreadFactoryImpl("MQ-AsyncArrayDispatcher-Thread" + traceInstanceId, true) + .newThread(new AsyncRunnable()); this.worker.setDaemon(true); this.worker.start(); this.registerShutDownHook(); @@ -197,37 +191,28 @@ public boolean append(final Object ctx) { @Override public void flush() { - // The maximum waiting time for refresh,avoid being written all the time, resulting in failure to return. long end = System.currentTimeMillis() + 500; - while (System.currentTimeMillis() <= end) { - synchronized (taskQueueByTopic) { - for (TraceDataSegment taskInfo : taskQueueByTopic.values()) { - taskInfo.sendAllData(); - } - } - synchronized (traceContextQueue) { - if (traceContextQueue.size() == 0 && appenderQueue.size() == 0) { - break; - } - } + while (traceContextQueue.size() > 0 || appenderQueue.size() > 0 && System.currentTimeMillis() <= end) { try { - Thread.sleep(1); - } catch (InterruptedException e) { - break; + flushTraceContext(true); + } catch (Throwable throwable) { + log.error("flushTraceContext error", throwable); } } - log.info("------end trace send " + traceContextQueue.size() + " " + appenderQueue.size()); + if (appenderQueue.size() > 0) { + log.error("There are still some traces that haven't been sent " + traceContextQueue.size() + " " + appenderQueue.size()); + } } @Override public void shutdown() { - this.stopped = true; flush(); this.traceExecutor.shutdown(); if (isStarted.get()) { traceProducer.shutdown(); } this.removeShutdownHook(); + stopped = true; } public void registerShutDownHook() { @@ -259,152 +244,111 @@ public void removeShutdownHook() { } class AsyncRunnable implements Runnable { - private boolean stopped; + private volatile boolean stopped = false; @Override public void run() { while (!stopped) { - synchronized (traceContextQueue) { - long endTime = System.currentTimeMillis() + pollingTimeMil; - while (System.currentTimeMillis() < endTime) { - try { - TraceContext traceContext = traceContextQueue.poll( - endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS - ); - - if (traceContext != null && !traceContext.getTraceBeans().isEmpty()) { - // get the topic which the trace message will send to - String traceTopicName = this.getTraceTopicName(traceContext.getRegionId()); - - // get the traceDataSegment which will save this trace message, create if null - TraceDataSegment traceDataSegment = taskQueueByTopic.get(traceTopicName); - if (traceDataSegment == null) { - traceDataSegment = new TraceDataSegment(traceTopicName, traceContext.getRegionId()); - taskQueueByTopic.put(traceTopicName, traceDataSegment); - } - - // encode traceContext and save it into traceDataSegment - // NOTE if data size in traceDataSegment more than maxMsgSize, - // a AsyncDataSendTask will be created and submitted - TraceTransferBean traceTransferBean = TraceDataEncoder.encoderFromContextBean(traceContext); - traceDataSegment.addTraceTransferBean(traceTransferBean); - } - } catch (InterruptedException ignore) { - log.debug("traceContextQueue#poll exception"); - } - } - - // NOTE send the data in traceDataSegment which the first TraceTransferBean - // is longer than waitTimeThreshold - sendDataByTimeThreshold(); - - if (AsyncTraceDispatcher.this.stopped) { - this.stopped = true; - } + try { + flushTraceContext(false); + } catch (Throwable e) { + log.error("flushTraceContext error", e); } } - + if (AsyncTraceDispatcher.this.stopped) { + this.stopped = true; + } } + } - private void sendDataByTimeThreshold() { - long now = System.currentTimeMillis(); - for (TraceDataSegment taskInfo : taskQueueByTopic.values()) { - if (now - taskInfo.firstBeanAddTime >= waitTimeThresholdMil) { - taskInfo.sendAllData(); + private void flushTraceContext(boolean forceFlush) throws InterruptedException { + List contextList = new ArrayList<>(batchNum); + int size = traceContextQueue.size(); + if (size != 0) { + if (forceFlush || size >= batchNum || System.currentTimeMillis() - lastFlushTime > flushTraceInterval) { + for (int i = 0; i < batchNum; i++) { + TraceContext context = traceContextQueue.poll(); + if (context != null) { + contextList.add(context); + } else { + break; + } } + asyncSendTraceMessage(contextList); + return; } } + // To prevent an infinite loop, add a wait time between each two task executions + Thread.sleep(5); + } - private String getTraceTopicName(String regionId) { - AccessChannel accessChannel = AsyncTraceDispatcher.this.getAccessChannel(); - if (AccessChannel.CLOUD == accessChannel) { - return TraceConstants.TRACE_TOPIC_PREFIX + regionId; - } - - return AsyncTraceDispatcher.this.getTraceTopicName(); - } + private void asyncSendTraceMessage(List contextList) { + AsyncDataSendTask request = new AsyncDataSendTask(contextList); + traceExecutor.submit(request); + lastFlushTime = System.currentTimeMillis(); } - class TraceDataSegment { - private long firstBeanAddTime; - private int currentMsgSize; - private int currentMsgKeySize; - private final String traceTopicName; - private final String regionId; - private final List traceTransferBeanList = new ArrayList<>(); + class AsyncDataSendTask implements Runnable { + private final List contextList; - TraceDataSegment(String traceTopicName, String regionId) { - this.traceTopicName = traceTopicName; - this.regionId = regionId; + public AsyncDataSendTask(List contextList) { + this.contextList = contextList; } - public void addTraceTransferBean(TraceTransferBean traceTransferBean) { - initFirstBeanAddTime(); - this.traceTransferBeanList.add(traceTransferBean); - this.currentMsgSize += traceTransferBean.getTransData().length(); - - this.currentMsgKeySize = traceTransferBean.getTransKey().stream() - .reduce(currentMsgKeySize, (acc, x) -> acc + x.length(), Integer::sum); - if (currentMsgSize >= traceProducer.getMaxMessageSize() - 10 * 1000 || currentMsgKeySize >= MAX_MSG_KEY_SIZE) { - List dataToSend = new ArrayList<>(traceTransferBeanList); - AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend); - traceExecutor.submit(asyncDataSendTask); - this.clear(); - } + @Override + public void run() { + sendTraceData(contextList); } - public void sendAllData() { - if (this.traceTransferBeanList.isEmpty()) { - return; + public void sendTraceData(List contextList) { + Map> transBeanMap = new HashMap<>(16); + String currentRegionId; + for (TraceContext context : contextList) { + currentRegionId = context.getRegionId(); + if (currentRegionId == null || context.getTraceBeans().isEmpty()) { + continue; + } + String topic = context.getTraceBeans().get(0).getTopic(); + String key = topic + TraceConstants.CONTENT_SPLITOR + currentRegionId; + List transBeanList = transBeanMap.computeIfAbsent(key, k -> new ArrayList<>()); + TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context); + transBeanList.add(traceData); } - List dataToSend = new ArrayList<>(traceTransferBeanList); - AsyncDataSendTask asyncDataSendTask = new AsyncDataSendTask(traceTopicName, regionId, dataToSend); - traceExecutor.submit(asyncDataSendTask); - - this.clear(); - } - - private void initFirstBeanAddTime() { - if (firstBeanAddTime == 0) { - firstBeanAddTime = System.currentTimeMillis(); + for (Map.Entry> entry : transBeanMap.entrySet()) { + String[] key = entry.getKey().split(String.valueOf(TraceConstants.CONTENT_SPLITOR)); + flushData(entry.getValue(), key[0], key[1]); } } - private void clear() { - this.firstBeanAddTime = 0; - this.currentMsgSize = 0; - this.currentMsgKeySize = 0; - this.traceTransferBeanList.clear(); - } - } - - class AsyncDataSendTask implements Runnable { - private final String traceTopicName; - private final String regionId; - private final List traceTransferBeanList; - - public AsyncDataSendTask(String traceTopicName, String regionId, List traceTransferBeanList) { - this.traceTopicName = traceTopicName; - this.regionId = regionId; - this.traceTransferBeanList = traceTransferBeanList; - } - - @Override - public void run() { + private void flushData(List transBeanList, String topic, String currentRegionId) { + if (transBeanList.size() == 0) { + return; + } StringBuilder buffer = new StringBuilder(1024); - Set keySet = new HashSet<>(); - for (TraceTransferBean bean : traceTransferBeanList) { + int count = 0; + Set keySet = new HashSet(); + for (TraceTransferBean bean : transBeanList) { keySet.addAll(bean.getTransKey()); buffer.append(bean.getTransData()); + count++; + if (buffer.length() >= traceProducer.getMaxMessageSize()) { + sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); + buffer.delete(0, buffer.length()); + keySet.clear(); + count = 0; + } + } + if (count > 0) { + sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); } - sendTraceDataByMQ(keySet, buffer.toString(), traceTopicName); + transBeanList.clear(); } /** * Send message trace data * - * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId) - * @param data the message trace data in this batch + * @param keySet the keyset in this batch(including msgId in original message not offsetMsgId) + * @param data the message trace data in this batch * @param traceTopic the topic which message trace data will send to */ private void sendTraceDataByMQ(Set keySet, final String data, String traceTopic) { @@ -467,4 +411,4 @@ private Set tryGetMessageQueueBrokerSet(DefaultMQProducerImpl producer, } } -} +} \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java index 0fdd95243a5..57e9b6410db 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -193,9 +193,10 @@ public static TraceTransferBean encoderFromContextBean(TraceContext ctx) { .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// .append(ctx.getContextCode()).append(TraceConstants.CONTENT_SPLITOR); if (!ctx.getAccessChannel().equals(AccessChannel.CLOUD)) { - sb.append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR) - .append(ctx.getGroupName()).append(TraceConstants.FIELD_SPLITOR); + sb.append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR); + sb.append(ctx.getGroupName()); } + sb.append(TraceConstants.FIELD_SPLITOR); } } break; From 2d44ec897c5ff9e1ce46a8ac8765c5cf493c7ac6 Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Mon, 29 Jul 2024 18:55:55 +0800 Subject: [PATCH 081/265] [ISSUE #8261] Avoid unnecessary waiting when a response is successfully returned (#8272) --- .../rocketmq/client/impl/producer/DefaultMQProducerImpl.java | 5 ++++- .../rocketmq/client/producer/RequestResponseFuture.java | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 7ef34025137..0e70ee25951 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -969,7 +969,7 @@ private SendResult sendKernelImpl(final Message msg, boolean messageCloned = false; if (msgBodyCompressed) { //If msg body was compressed, msgbody should be reset using prevBody. - //Clone new message using commpressed message body and recover origin massage. + //Clone new message using compressed message body and recover origin massage. //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 tmpMessage = MessageAccessor.cloneMessage(msg); messageCloned = true; @@ -1538,6 +1538,7 @@ public Message request(final Message msg, @Override public void onSuccess(SendResult sendResult) { requestResponseFuture.setSendRequestOk(true); + requestResponseFuture.acquireCountDownLatch(); } @Override @@ -1595,6 +1596,7 @@ public Message request(final Message msg, final MessageQueueSelector selector, f @Override public void onSuccess(SendResult sendResult) { requestResponseFuture.setSendRequestOk(true); + requestResponseFuture.acquireCountDownLatch(); } @Override @@ -1652,6 +1654,7 @@ public Message request(final Message msg, final MessageQueue mq, final long time @Override public void onSuccess(SendResult sendResult) { requestResponseFuture.setSendRequestOk(true); + requestResponseFuture.acquireCountDownLatch(); } @Override diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java index e66c08fdc53..203f92907a4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -107,6 +107,10 @@ public void setSendRequestOk(boolean sendRequestOk) { this.sendRequestOk = sendRequestOk; } + public void acquireCountDownLatch() { + this.countDownLatch.countDown(); + } + public Message getRequestMsg() { return requestMsg; } From ce27e1fc643d4b6a47b7a63784fab7e8070322e9 Mon Sep 17 00:00:00 2001 From: cserwen Date: Tue, 30 Jul 2024 09:11:55 +0800 Subject: [PATCH 082/265] [ISSUE #8332] fix: ack msg which has reached maxReconsumeTimes Co-authored-by: dengzhiwen1 --- .../ConsumeMessagePopConcurrentlyService.java | 2 +- .../impl/consumer/DefaultMQPushConsumerImpl.java | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java index 3713d1aba4d..d5191871106 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyService.java @@ -471,7 +471,7 @@ public void run() { processQueue.decFoundMsg(-msgs.size()); } - log.warn("processQueue invalid. isDropped={}, isPopTimeout={}, messageQueue={}, msgs={}", + log.warn("processQueue invalid or popTimeout. isDropped={}, isPopTimeout={}, messageQueue={}, msgs={}", processQueue.isDropped(), isPopTimeout(), messageQueue, msgs); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 3e832e5a9a3..e66a9825f3d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -621,10 +621,9 @@ public void onException(Throwable e) { private PopResult processPopResult(final PopResult popResult, final SubscriptionData subscriptionData) { if (PopStatus.FOUND == popResult.getPopStatus()) { List msgFoundList = popResult.getMsgFoundList(); - List msgListFilterAgain = msgFoundList; + List msgListFilterAgain = new ArrayList<>(popResult.getMsgFoundList().size()); if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode() && popResult.getMsgFoundList().size() > 0) { - msgListFilterAgain = new ArrayList<>(popResult.getMsgFoundList().size()); for (MessageExt msg : popResult.getMsgFoundList()) { if (msg.getTags() != null) { if (subscriptionData.getTagsSet().contains(msg.getTags())) { @@ -632,6 +631,8 @@ private PopResult processPopResult(final PopResult popResult, final Subscription } } } + } else { + msgListFilterAgain.addAll(msgFoundList); } if (!this.filterMessageHookList.isEmpty()) { @@ -649,6 +650,15 @@ private PopResult processPopResult(final PopResult popResult, final Subscription } } + Iterator iterator = msgListFilterAgain.iterator(); + while (iterator.hasNext()) { + MessageExt msg = iterator.next(); + if (msg.getReconsumeTimes() > defaultMQPushConsumer.getMaxReconsumeTimes()) { + iterator.remove(); + log.info("Reconsume times has reached {}, so ack msg={}", msg.getReconsumeTimes(), msg); + } + } + if (msgFoundList.size() != msgListFilterAgain.size()) { for (MessageExt msg : msgFoundList) { if (!msgListFilterAgain.contains(msg)) { From f1fe4bf754f277c6617269946eb0c5a6a0b09ed7 Mon Sep 17 00:00:00 2001 From: Z F <1096024838@qq.com> Date: Tue, 30 Jul 2024 09:12:29 +0800 Subject: [PATCH 083/265] [ISSUE #7731] Fix windows runBroker.cmd cannot start (#7731) (#8338) Co-authored-by: fz --- distribution/bin/runbroker.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd index 0ea87f876db..fefaab3013f 100644 --- a/distribution/bin/runbroker.cmd +++ b/distribution/bin/runbroker.cmd @@ -23,7 +23,7 @@ set BASE_DIR=%~dp0 set BASE_DIR=%BASE_DIR:~0,-1% for %%d in (%BASE_DIR%) do set BASE_DIR=%%~dpd -set CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\*;%CLASSPATH% +set CLASSPATH=.;%BASE_DIR%conf;%BASE_DIR%lib\*;"%CLASSPATH%" rem =========================================================================================== rem JVM Configuration From 8fe3451311ac499602406bf383cd4d7397732ff1 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 31 Jul 2024 10:56:17 +0800 Subject: [PATCH 084/265] [ISSUE #8465] Add more test coverage for ConsumeMessagePopConcurrentlyService (#8466) --- ...sumeMessagePopConcurrentlyServiceTest.java | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java new file mode 100644 index 00000000000..5097f14ca34 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopConcurrentlyServiceTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.stat.ConsumerStatsManager; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.body.CMResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ConsumeMessagePopConcurrentlyServiceTest { + + @Mock + private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + + @Mock + private MessageListenerConcurrently messageListener; + + @Mock + private DefaultMQPushConsumer defaultMQPushConsumer; + + private ConsumeMessagePopConcurrentlyService popService; + + private final String defaultGroup = "defaultGroup"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultTopic = "defaultTopic"; + + @Before + public void init() throws Exception { + when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup); + when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1); + when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3); + when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(32); + when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer); + ConsumerStatsManager consumerStatsManager = mock(ConsumerStatsManager.class); + when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager); + popService = new ConsumeMessagePopConcurrentlyService(defaultMQPushConsumerImpl, messageListener); + } + + @Test + public void testUpdateCorePoolSize() { + popService.updateCorePoolSize(2); + popService.incCorePoolSize(); + popService.decCorePoolSize(); + assertEquals(2, popService.getCorePoolSize()); + } + + @Test + public void testConsumeMessageDirectly() { + when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.CONSUME_SUCCESS); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult()); + } + + @Test + public void testConsumeMessageDirectlyWithCrLater() { + when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenReturn(ConsumeConcurrentlyStatus.RECONSUME_LATER); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_LATER, actual.getConsumeResult()); + } + + @Test + public void testConsumeMessageDirectlyWithCrReturnNull() { + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult()); + } + + @Test + public void testConsumeMessageDirectlyWithCrThrowException() { + when(messageListener.consumeMessage(any(), any(ConsumeConcurrentlyContext.class))).thenThrow(new RuntimeException("exception")); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult()); + } + + @Test + public void testShutdown() throws IllegalAccessException { + popService.shutdown(3000L); + Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), "scheduledExecutorService", true); + Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), "consumeExecutor", true); + ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService); + ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService); + assertTrue(scheduledExecutorService.isShutdown()); + assertTrue(scheduledExecutorService.isTerminated()); + assertTrue(consumeExecutor.isShutdown()); + assertTrue(consumeExecutor.isTerminated()); + } + + @Test + public void testSubmitConsumeRequest() { + assertThrows(UnsupportedOperationException.class, () -> { + List msgs = mock(List.class); + ProcessQueue processQueue = mock(ProcessQueue.class); + MessageQueue messageQueue = mock(MessageQueue.class); + popService.submitConsumeRequest(msgs, processQueue, messageQueue, false); + }); + } + + @Test + public void testSubmitPopConsumeRequest() throws IllegalAccessException { + List msgs = Collections.singletonList(createMessageExt()); + PopProcessQueue processQueue = mock(PopProcessQueue.class); + MessageQueue messageQueue = mock(MessageQueue.class); + ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class); + FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true); + popService.submitPopConsumeRequest(msgs, processQueue, messageQueue); + verify(consumeExecutor, times(1)).submit(any(Runnable.class)); + } + + @Test + public void testSubmitPopConsumeRequestWithMultiMsg() throws IllegalAccessException { + List msgs = Arrays.asList(createMessageExt(), createMessageExt()); + PopProcessQueue processQueue = mock(PopProcessQueue.class); + MessageQueue messageQueue = mock(MessageQueue.class); + ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class); + FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true); + when(defaultMQPushConsumer.getConsumeMessageBatchMaxSize()).thenReturn(1); + popService.submitPopConsumeRequest(msgs, processQueue, messageQueue); + verify(consumeExecutor, times(2)).submit(any(Runnable.class)); + } + + @Test + public void testProcessConsumeResult() { + ConsumeConcurrentlyContext context = mock(ConsumeConcurrentlyContext.class); + ConsumeMessagePopConcurrentlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopConcurrentlyService.ConsumeRequest.class); + when(consumeRequest.getMsgs()).thenReturn(Arrays.asList(createMessageExt(), createMessageExt())); + MessageQueue messageQueue = mock(MessageQueue.class); + when(messageQueue.getTopic()).thenReturn(defaultTopic); + when(consumeRequest.getMessageQueue()).thenReturn(messageQueue); + PopProcessQueue processQueue = mock(PopProcessQueue.class); + when(processQueue.ack()).thenReturn(0); + when(consumeRequest.getPopProcessQueue()).thenReturn(processQueue); + when(defaultMQPushConsumerImpl.getPopDelayLevel()).thenReturn(new int[]{1, 10}); + popService.processConsumeResult(ConsumeConcurrentlyStatus.CONSUME_SUCCESS, context, consumeRequest); + verify(defaultMQPushConsumerImpl, times(1)).ackAsync(any(MessageExt.class), any()); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + long curTime = System.currentTimeMillis(); + result.setBornTimestamp(curTime - 1000); + result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } +} From 69334a708cb303893a3d52bc026b3a466545053e Mon Sep 17 00:00:00 2001 From: rongtong Date: Thu, 1 Aug 2024 10:01:50 +0800 Subject: [PATCH 085/265] [ISSUE #8432] Make autoDeleteUnusedStats default to true --- .../src/main/java/org/apache/rocketmq/common/BrokerConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 3aac59e0a1b..8982e59d03e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -288,7 +288,7 @@ public class BrokerConfig extends BrokerIdentity { private boolean enableDetailStat = true; - private boolean autoDeleteUnusedStats = false; + private boolean autoDeleteUnusedStats = true; /** * Whether to distinguish log paths when multiple brokers are deployed on the same machine From 3696be06321c24b4c534a5a6299fd587710b5de4 Mon Sep 17 00:00:00 2001 From: rongtong Date: Thu, 1 Aug 2024 10:04:11 +0800 Subject: [PATCH 086/265] [ISSUE #8463] Some statistical items should also be deleted to prevent memory leakage when a topic or group is deleted (#8464) * Some important statistical items should also be deleted to prevent memory leakage when a topic or group is deleted * Add UTs --- .../apache/rocketmq/common/stats/Stats.java | 3 +++ .../store/stats/BrokerStatsManager.java | 24 +++++++++++-------- .../store/stats/BrokerStatsManagerTest.java | 13 ++++++++++ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java b/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java index b70f96e412e..f67ccf9ae90 100644 --- a/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java +++ b/common/src/main/java/org/apache/rocketmq/common/stats/Stats.java @@ -44,4 +44,7 @@ public class Stats { public static final String GROUP_GET_FALL_SIZE = "GROUP_GET_FALL_SIZE"; public static final String GROUP_GET_FALL_TIME = "GROUP_GET_FALL_TIME"; public static final String GROUP_GET_LATENCY = "GROUP_GET_LATENCY"; + public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY"; + public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS"; + public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS"; } diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java index c165d333fd0..a6c33f61311 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java @@ -69,9 +69,9 @@ public class BrokerStatsManager { @Deprecated public static final String COMMERCIAL_PERM_FAILURES = Stats.COMMERCIAL_PERM_FAILURES; // Send message latency - public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY"; - public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS"; - public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS"; + @Deprecated public static final String TOPIC_PUT_LATENCY = "TOPIC_PUT_LATENCY"; + @Deprecated public static final String GROUP_ACK_NUMS = "GROUP_ACK_NUMS"; + @Deprecated public static final String GROUP_CK_NUMS = "GROUP_CK_NUMS"; public static final String DLQ_PUT_NUMS = "DLQ_PUT_NUMS"; public static final String BROKER_ACK_NUMS = "BROKER_ACK_NUMS"; public static final String BROKER_CK_NUMS = "BROKER_CK_NUMS"; @@ -179,10 +179,10 @@ public void init() { this.statsTable.put(Stats.TOPIC_PUT_SIZE, new StatsItemSet(Stats.TOPIC_PUT_SIZE, this.scheduledExecutorService, log)); this.statsTable.put(Stats.GROUP_GET_NUMS, new StatsItemSet(Stats.GROUP_GET_NUMS, this.scheduledExecutorService, log)); this.statsTable.put(Stats.GROUP_GET_SIZE, new StatsItemSet(Stats.GROUP_GET_SIZE, this.scheduledExecutorService, log)); - this.statsTable.put(GROUP_ACK_NUMS, new StatsItemSet(GROUP_ACK_NUMS, this.scheduledExecutorService, log)); - this.statsTable.put(GROUP_CK_NUMS, new StatsItemSet(GROUP_CK_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(Stats.GROUP_ACK_NUMS, new StatsItemSet(Stats.GROUP_ACK_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(Stats.GROUP_CK_NUMS, new StatsItemSet(Stats.GROUP_CK_NUMS, this.scheduledExecutorService, log)); this.statsTable.put(Stats.GROUP_GET_LATENCY, new StatsItemSet(Stats.GROUP_GET_LATENCY, this.scheduledExecutorService, log)); - this.statsTable.put(TOPIC_PUT_LATENCY, new StatsItemSet(TOPIC_PUT_LATENCY, this.scheduledExecutorService, log)); + this.statsTable.put(Stats.TOPIC_PUT_LATENCY, new StatsItemSet(Stats.TOPIC_PUT_LATENCY, this.scheduledExecutorService, log)); this.statsTable.put(Stats.SNDBCK_PUT_NUMS, new StatsItemSet(Stats.SNDBCK_PUT_NUMS, this.scheduledExecutorService, log)); this.statsTable.put(DLQ_PUT_NUMS, new StatsItemSet(DLQ_PUT_NUMS, this.scheduledExecutorService, log)); this.statsTable.put(Stats.BROKER_PUT_NUMS, new StatsItemSet(Stats.BROKER_PUT_NUMS, this.scheduledExecutorService, log)); @@ -338,10 +338,13 @@ public void onTopicDeleted(final String topic) { } this.statsTable.get(Stats.GROUP_GET_NUMS).delValueByPrefixKey(topic, "@"); this.statsTable.get(Stats.GROUP_GET_SIZE).delValueByPrefixKey(topic, "@"); + this.statsTable.get(Stats.GROUP_CK_NUMS).delValueByPrefixKey(topic, "@"); + this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueByPrefixKey(topic, "@"); this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueByPrefixKey(topic, "@"); this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueByPrefixKey(topic, "@"); this.statsTable.get(Stats.SNDBCK_PUT_NUMS).delValueByPrefixKey(topic, "@"); this.statsTable.get(Stats.GROUP_GET_LATENCY).delValueByInfixKey(topic, "@"); + this.statsTable.get(Stats.TOPIC_PUT_LATENCY).delValueBySuffixKey(topic, "@"); this.momentStatsItemSetFallSize.delValueByInfixKey(topic, "@"); this.momentStatsItemSetFallTime.delValueByInfixKey(topic, "@"); } @@ -349,6 +352,8 @@ public void onTopicDeleted(final String topic) { public void onGroupDeleted(final String group) { this.statsTable.get(Stats.GROUP_GET_NUMS).delValueBySuffixKey(group, "@"); this.statsTable.get(Stats.GROUP_GET_SIZE).delValueBySuffixKey(group, "@"); + this.statsTable.get(Stats.GROUP_CK_NUMS).delValueBySuffixKey(group, "@"); + this.statsTable.get(Stats.GROUP_ACK_NUMS).delValueBySuffixKey(group, "@"); if (enableQueueStat) { this.statsTable.get(Stats.QUEUE_GET_NUMS).delValueBySuffixKey(group, "@"); this.statsTable.get(Stats.QUEUE_GET_SIZE).delValueBySuffixKey(group, "@"); @@ -434,12 +439,12 @@ public void incGroupGetNums(final String group, final String topic, final int in public void incGroupCkNums(final String group, final String topic, final int incValue) { final String statsKey = buildStatsKey(topic, group); - this.statsTable.get(GROUP_CK_NUMS).addValue(statsKey, incValue, 1); + this.statsTable.get(Stats.GROUP_CK_NUMS).addValue(statsKey, incValue, 1); } public void incGroupAckNums(final String group, final String topic, final int incValue) { final String statsKey = buildStatsKey(topic, group); - this.statsTable.get(GROUP_ACK_NUMS).addValue(statsKey, incValue, 1); + this.statsTable.get(Stats.GROUP_ACK_NUMS).addValue(statsKey, incValue, 1); } public String buildStatsKey(String topic, String group) { @@ -509,9 +514,8 @@ public void incTopicPutLatency(final String topic, final int queueId, final int statsKey = new StringBuilder(6); } statsKey.append(queueId).append("@").append(topic); - this.statsTable.get(TOPIC_PUT_LATENCY).addValue(statsKey.toString(), incValue, 1); + this.statsTable.get(Stats.TOPIC_PUT_LATENCY).addValue(statsKey.toString(), incValue, 1); } - public void incBrokerPutNums() { this.statsTable.get(Stats.BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().add(1); } diff --git a/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java b/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java index a602da09395..058ad0b0208 100644 --- a/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/stats/BrokerStatsManagerTest.java @@ -24,6 +24,8 @@ import org.junit.Test; import static org.apache.rocketmq.common.stats.Stats.BROKER_PUT_NUMS; +import static org.apache.rocketmq.common.stats.Stats.GROUP_ACK_NUMS; +import static org.apache.rocketmq.common.stats.Stats.GROUP_CK_NUMS; import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_SIZE; import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_FALL_TIME; import static org.apache.rocketmq.common.stats.Stats.GROUP_GET_LATENCY; @@ -34,6 +36,7 @@ import static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_NUMS; import static org.apache.rocketmq.common.stats.Stats.QUEUE_PUT_SIZE; import static org.apache.rocketmq.common.stats.Stats.SNDBCK_PUT_NUMS; +import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_LATENCY; import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_NUMS; import static org.apache.rocketmq.common.stats.Stats.TOPIC_PUT_SIZE; import static org.assertj.core.api.Assertions.assertThat; @@ -139,8 +142,11 @@ public void testOnTopicDeleted() { brokerStatsManager.incTopicPutSize(TOPIC, 100); brokerStatsManager.incQueuePutNums(TOPIC, QUEUE_ID); brokerStatsManager.incQueuePutSize(TOPIC, QUEUE_ID, 100); + brokerStatsManager.incTopicPutLatency(TOPIC, QUEUE_ID, 10); brokerStatsManager.incGroupGetNums(GROUP_NAME, TOPIC, 1); brokerStatsManager.incGroupGetSize(GROUP_NAME, TOPIC, 100); + brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1); + brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1); brokerStatsManager.incQueueGetNums(GROUP_NAME, TOPIC, QUEUE_ID, 1); brokerStatsManager.incQueueGetSize(GROUP_NAME, TOPIC, QUEUE_ID, 100); brokerStatsManager.incSendBackNums(GROUP_NAME, TOPIC); @@ -162,6 +168,9 @@ public void testOnTopicDeleted() { Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME)); Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME)); Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, "1@" + TOPIC + "@" + GROUP_NAME)); + Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + "@" + GROUP_NAME)); + Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + "@" + GROUP_NAME)); + Assert.assertNull(brokerStatsManager.getStatsItem(TOPIC_PUT_LATENCY, QUEUE_ID + "@" + TOPIC)); } @Test @@ -174,6 +183,8 @@ public void testOnGroupDeleted() { brokerStatsManager.incGroupGetLatency(GROUP_NAME, TOPIC, 1, 1); brokerStatsManager.recordDiskFallBehindTime(GROUP_NAME, TOPIC, 1, 11L); brokerStatsManager.recordDiskFallBehindSize(GROUP_NAME, TOPIC, 1, 11L); + brokerStatsManager.incGroupCkNums(GROUP_NAME, TOPIC, 1); + brokerStatsManager.incGroupAckNums(GROUP_NAME, TOPIC, 1); brokerStatsManager.onGroupDeleted(GROUP_NAME); @@ -185,6 +196,8 @@ public void testOnGroupDeleted() { Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_LATENCY, "1@" + TOPIC + "@" + GROUP_NAME)); Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_SIZE, "1@" + TOPIC + "@" + GROUP_NAME)); Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_GET_FALL_TIME, "1@" + TOPIC + "@" + GROUP_NAME)); + Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_CK_NUMS, TOPIC + "@" + GROUP_NAME)); + Assert.assertNull(brokerStatsManager.getStatsItem(GROUP_ACK_NUMS, TOPIC + "@" + GROUP_NAME)); } @Test From cd23d6d0d646079b75f95c3f9d125473b6305018 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 1 Aug 2024 10:06:53 +0800 Subject: [PATCH 087/265] [ISSUE #8472] Fix pop message delay due to not notify message arriving after suspend (#8473) --- .../rocketmq/broker/processor/PopMessageProcessor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 89b4c39d72b..6073023722a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -431,6 +431,11 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC PollingResult pollingResult = popLongPollingService.polling( ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter); if (PollingResult.POLLING_SUC == pollingResult) { + if (restNum > 0) { + popLongPollingService.notifyMessageArriving( + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(), + null, 0L, null, null); + } return null; } else if (PollingResult.POLLING_FULL == pollingResult) { finalResponse.setCode(ResponseCode.POLLING_FULL); From 2ed4ba23e0448d5a7a4f68c29d0f9bb7a6d30ee3 Mon Sep 17 00:00:00 2001 From: TestBoost <153348110+TestBoost@users.noreply.github.com> Date: Thu, 1 Aug 2024 02:17:57 -0500 Subject: [PATCH 088/265] nly initialize all the variables once to speed up test ConsumeMessageConcurrentlyServiceTest (#8436) --- ...ConsumeMessageConcurrentlyServiceTest.java | 123 ++++++++---------- 1 file changed, 54 insertions(+), 69 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java index 749201e3c22..395c0ff2335 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java @@ -52,15 +52,14 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.body.ConsumeStatus; import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; -import org.junit.After; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -70,49 +69,54 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import org.mockito.Mockito; @RunWith(MockitoJUnitRunner.class) public class ConsumeMessageConcurrentlyServiceTest { - private String consumerGroup; - private String topic = "FooBar"; - private String brokerName = "BrokerA"; - private MQClientInstance mQClientFactory; + + private static String consumerGroup; + + private static String topic = "FooBar"; + + private static String brokerName = "BrokerA"; + + private static MQClientInstance mQClientFactory; @Mock - private MQClientAPIImpl mQClientAPIImpl; - private PullAPIWrapper pullAPIWrapper; - private RebalancePushImpl rebalancePushImpl; - private DefaultMQPushConsumer pushConsumer; + private static MQClientAPIImpl mQClientAPIImpl; + + private static PullAPIWrapper pullAPIWrapper; + + private static RebalancePushImpl rebalancePushImpl; + + private static DefaultMQPushConsumer pushConsumer; - @Before - public void init() throws Exception { + @BeforeClass + public static void init() throws Exception { + mQClientAPIImpl = Mockito.mock(MQClientAPIImpl.class); ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); Collection instances = factoryTable.values(); for (MQClientInstance instance : instances) { instance.shutdown(); } factoryTable.clear(); - consumerGroup = "FooBarGroup" + System.currentTimeMillis(); pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.setNamesrvAddr("127.0.0.1:9876"); pushConsumer.setPullInterval(60 * 1000); - pushConsumer.registerMessageListener(new MessageListenerConcurrently() { + @Override - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }); - DefaultMQPushConsumerImpl pushConsumerImpl = pushConsumer.getDefaultMQPushConsumerImpl(); rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); field.set(pushConsumerImpl, rebalancePushImpl); pushConsumer.subscribe(topic, "*"); - // suppress updateTopicRouteInfoFromNameServer pushConsumer.changeInstanceNameToPID(); mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); @@ -121,38 +125,32 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, field.setAccessible(true); field.set(pushConsumerImpl, mQClientFactory); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); - field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); field.setAccessible(true); field.set(mQClientFactory, mQClientAPIImpl); - pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); field.setAccessible(true); field.set(pushConsumerImpl, pullAPIWrapper); - pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory); + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))).thenAnswer(new Answer() { - when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), - anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { - @Override - public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - MessageClientExt messageClientExt = new MessageClientExt(); - messageClientExt.setTopic(topic); - messageClientExt.setQueueId(0); - messageClientExt.setMsgId("123"); - messageClientExt.setBody(new byte[] {'a'}); - messageClientExt.setOffsetMsgId("234"); - messageClientExt.setBornHost(new InetSocketAddress(8080)); - messageClientExt.setStoreHost(new InetSocketAddress(8080)); - PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); - ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); - return pullResult; - } - }); - + @Override + public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] { 'a' }); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); + return pullResult; + } + }); doReturn(new FindBrokerResult("127.0.0.1:10912", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); Set messageQueueSet = new HashSet<>(); @@ -162,54 +160,45 @@ public PullResult answer(InvocationOnMock mock) throws Throwable { } @Test - public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception { + public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException, Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicReference messageAtomic = new AtomicReference<>(); + ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { - ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { messageAtomic.set(msgs.get(0)); countDownLatch.countDown(); return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie); - PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); pullMessageService.executePullRequestImmediately(createPullRequest()); countDownLatch.await(); - Thread.sleep(1000); - - ConsumeStatus stats = normalServie.getConsumerStatsManager().consumeStatus(pushConsumer.getDefaultMQPushConsumerImpl().groupName(),topic); - - ConsumerStatsManager mgr = normalServie.getConsumerStatsManager(); - + ConsumeStatus stats = normalServie.getConsumerStatsManager().consumeStatus(pushConsumer.getDefaultMQPushConsumerImpl().groupName(), topic); + ConsumerStatsManager mgr = normalServie.getConsumerStatsManager(); Field statItmeSetField = mgr.getClass().getDeclaredField("topicAndGroupConsumeOKTPS"); statItmeSetField.setAccessible(true); - - StatsItemSet itemSet = (StatsItemSet)statItmeSetField.get(mgr); + StatsItemSet itemSet = (StatsItemSet) statItmeSetField.get(mgr); StatsItem item = itemSet.getAndCreateStatsItem(topic + "@" + pushConsumer.getDefaultMQPushConsumerImpl().groupName()); - assertThat(item.getValue().sum()).isGreaterThan(0L); MessageExt msg = messageAtomic.get(); assertThat(msg).isNotNull(); assertThat(msg.getTopic()).isEqualTo(topic); - assertThat(msg.getBody()).isEqualTo(new byte[] {'a'}); + assertThat(msg.getBody()).isEqualTo(new byte[] { 'a' }); } - @After - public void terminate() { + @AfterClass + public static void terminate() { pushConsumer.shutdown(); } - private PullRequest createPullRequest() { + private static PullRequest createPullRequest() { PullRequest pullRequest = new PullRequest(); pullRequest.setConsumerGroup(consumerGroup); pullRequest.setNextOffset(1024); - MessageQueue messageQueue = new MessageQueue(); messageQueue.setBrokerName(brokerName); messageQueue.setQueueId(0); @@ -219,12 +208,10 @@ private PullRequest createPullRequest() { processQueue.setLocked(true); processQueue.setLastLockTimestamp(System.currentTimeMillis()); pullRequest.setProcessQueue(processQueue); - return pullRequest; } - private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, - List messageExtList) throws Exception { + private static PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, List messageExtList) throws Exception { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); for (MessageExt messageExt : messageExtList) { outputStream.write(MessageDecoder.encode(messageExt, false)); @@ -236,23 +223,21 @@ private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, P public void testConsumeThreadName() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicReference consumeThreadName = new AtomicReference<>(); - StringBuilder consumeGroup2 = new StringBuilder(); for (int i = 0; i < 101; i++) { consumeGroup2.append(i).append("#"); } pushConsumer.setConsumerGroup(consumeGroup2.toString()); - ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { + ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { + @Override - public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { consumeThreadName.set(Thread.currentThread().getName()); countDownLatch.countDown(); return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } }); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(normalServie); - PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); pullMessageService.executePullRequestImmediately(createPullRequest()); countDownLatch.await(); From aab646ce83930454d8b5956779aa28a80e326e24 Mon Sep 17 00:00:00 2001 From: imzs Date: Fri, 2 Aug 2024 14:43:21 +0800 Subject: [PATCH 089/265] [ISSUE #8460] Improve the pop revive process when reading biz messages from a remote broker (#8475) * [ISSUE #8460] part1: add extra information to the call chain of remote message reading * [ISSUE #8460] part2: add exponential backoff and ending condition of CK rewrite, and fix checkstyle * [ISSUE #8460] exclude test, BrokerOuterAPITest passed locally, but failed on bazel. --- broker/BUILD.bazel | 1 + .../broker/failover/EscapeBridge.java | 44 +++-- .../rocketmq/broker/out/BrokerOuterAPI.java | 13 +- .../broker/processor/PopReviveService.java | 41 +++-- .../rocketmq/broker/BrokerOuterAPITest.java | 173 +++++++++++++++++- .../broker/failover/EscapeBridgeTest.java | 150 ++++++++++++++- .../processor/PopReviveServiceTest.java | 166 ++++++++++++++++- .../rocketmq/store/pop/PopCheckPoint.java | 23 ++- 8 files changed, 554 insertions(+), 57 deletions(-) diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 0dbc85f9453..66e621e9301 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -100,6 +100,7 @@ GenTestRules( exclude_tests = [ # These tests are extremely slow and flaky, exclude them before they are properly fixed. "src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest", + "src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest", ], deps = [ ":tests", diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java index ededaf2c65e..7df49f8c470 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java @@ -25,7 +25,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; import org.apache.rocketmq.client.consumer.PullStatus; @@ -34,7 +36,6 @@ import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageConst; @@ -47,7 +48,6 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.store.GetMessageResult; -import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; @@ -263,34 +263,29 @@ private PutMessageResult transformSendResult2PutResult(SendResult sendResult) { } } - public Pair getMessage(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { + public Triple getMessage(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { return getMessageAsync(topic, offset, queueId, brokerName, deCompressBody).join(); } - public CompletableFuture> getMessageAsync(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { + // Triple, check info and retry if and only if MessageExt is null + public CompletableFuture> getMessageAsync(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName); if (messageStore != null) { return messageStore.getMessageAsync(innerConsumerGroupName, topic, queueId, offset, 1, null) .thenApply(result -> { if (result == null) { LOG.warn("getMessageResult is null , innerConsumerGroupName {}, topic {}, offset {}, queueId {}", innerConsumerGroupName, topic, offset, queueId); - return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null); + return Triple.of(null, "getMessageResult is null", false); // local store, so no retry } List list = decodeMsgList(result, deCompressBody); if (list == null || list.isEmpty()) { LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, result); - return new Pair<>(result.getStatus(), null); + return Triple.of(null, "Can not get msg", false); // local store, so no retry } - return new Pair<>(result.getStatus(), list.get(0)); + return Triple.of(list.get(0), "", false); }); } else { - return getMessageFromRemoteAsync(topic, offset, queueId, brokerName) - .thenApply(msg -> { - if (msg == null) { - return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null); - } - return new Pair<>(GetMessageStatus.FOUND, msg); - }); + return getMessageFromRemoteAsync(topic, offset, queueId, brokerName); } } @@ -322,11 +317,12 @@ protected List decodeMsgList(GetMessageResult getMessageResult, bool return foundList; } - protected MessageExt getMessageFromRemote(String topic, long offset, int queueId, String brokerName) { + protected Triple getMessageFromRemote(String topic, long offset, int queueId, String brokerName) { return getMessageFromRemoteAsync(topic, offset, queueId, brokerName).join(); } - protected CompletableFuture getMessageFromRemoteAsync(String topic, long offset, int queueId, String brokerName) { + // Triple, check info and retry if and only if MessageExt is null + protected CompletableFuture> getMessageFromRemoteAsync(String topic, long offset, int queueId, String brokerName) { try { String brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false); if (null == brokerAddr) { @@ -334,23 +330,25 @@ protected CompletableFuture getMessageFromRemoteAsync(String topic, brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false); if (null == brokerAddr) { - LOG.warn("can't find broker address for topic {}", topic); - return CompletableFuture.completedFuture(null); + LOG.warn("can't find broker address for topic {}, {}", topic, brokerName); + return CompletableFuture.completedFuture(Triple.of(null, "brokerAddress not found", true)); // maybe offline temporarily, so need retry } } return this.brokerController.getBrokerOuterAPI().pullMessageFromSpecificBrokerAsync(brokerName, brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS) .thenApply(pullResult -> { - if (pullResult.getPullStatus().equals(PullStatus.FOUND) && !pullResult.getMsgFoundList().isEmpty()) { - return pullResult.getMsgFoundList().get(0); + if (pullResult.getLeft() != null + && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus()) + && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) { + return Triple.of(pullResult.getLeft().getMsgFoundList().get(0), "", false); } - return null; + return Triple.of(null, pullResult.getMiddle(), pullResult.getRight()); }); } catch (Exception e) { - LOG.error("Get message from remote failed.", e); + LOG.error("Get message from remote failed. {}, {}, {}, {}", topic, offset, queueId, brokerName, e); } - return CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(Triple.of(null, "Get message from remote failed", true)); // need retry } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index d5c80ce2ec3..83edd88408a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.acl.common.AclClientRPCHook; import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.auth.config.AuthConfig; @@ -1378,7 +1379,8 @@ public void run0() { }); } - public CompletableFuture pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr, + // Triple, should check info and retry if and only if PullResult is null + public CompletableFuture> pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr, String consumerGroup, String topic, int queueId, long offset, int maxNums, long timeoutMillis) throws RemotingException, InterruptedException { PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); @@ -1397,7 +1399,7 @@ public CompletableFuture pullMessageFromSpecificBrokerAsync(String b requestHeader.setBrokerName(brokerName); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); - CompletableFuture pullResultFuture = new CompletableFuture<>(); + CompletableFuture> pullResultFuture = new CompletableFuture<>(); this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { @@ -1409,15 +1411,16 @@ public void operationSucceed(RemotingCommand response) { try { PullResultExt pullResultExt = processPullResponse(response, brokerAddr); processPullResult(pullResultExt, brokerName, queueId); - pullResultFuture.complete(pullResultExt); + pullResultFuture.complete(Triple.of(pullResultExt, pullResultExt.getPullStatus().name(), false)); // found or not found really, so no retry } catch (Exception e) { - pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); + // retry when NO_PERMISSION, SUBSCRIPTION_GROUP_NOT_EXIST etc. even when TOPIC_NOT_EXIST + pullResultFuture.complete(Triple.of(null, "Response Code:" + response.getCode(), true)); } } @Override public void operationFail(Throwable throwable) { - pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); + pullResultFuture.complete(Triple.of(null, throwable.getMessage(), true)); } }); return pullResultFuture; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 8074af23bfe..114d094600e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -27,6 +27,7 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; import org.apache.rocketmq.broker.metrics.PopMetricsManager; @@ -34,6 +35,7 @@ import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PopAckConstants; import org.apache.rocketmq.common.ServiceThread; @@ -51,7 +53,6 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.GetMessageResult; -import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; @@ -63,6 +64,7 @@ public class PopReviveService extends ServiceThread { private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + private final int[] ckRewriteIntervalsInSeconds = new int[] { 10, 20, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200 }; private int queueId; private BrokerController brokerController; @@ -196,7 +198,8 @@ private boolean reachTail(PullResult pullResult, long offset) { || pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset(); } - private CompletableFuture> getBizMessage(String topic, long offset, int queueId, + // Triple + private CompletableFuture> getBizMessage(String topic, long offset, int queueId, String brokerName) { return this.brokerController.getEscapeBridge().getMessageAsync(topic, offset, queueId, brokerName, false); } @@ -491,6 +494,8 @@ protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwabl PopCheckPoint oldCK = inflightReviveRequestMap.firstKey(); rePutCK(oldCK, pair); inflightReviveRequestMap.remove(oldCK); + POP_LOGGER.warn("stay too long, remove from reviveRequestMap, {}, {}, {}, {}", popCheckPoint.getTopic(), + popCheckPoint.getBrokerName(), popCheckPoint.getQueueId(), popCheckPoint.getStartOffset()); } } @@ -524,22 +529,12 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { // retry msg long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j); CompletableFuture> future = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName()) - .thenApply(resultPair -> { - GetMessageStatus getMessageStatus = resultPair.getObject1(); - MessageExt message = resultPair.getObject2(); + .thenApply(rst -> { + MessageExt message = rst.getLeft(); if (message == null) { - POP_LOGGER.debug("reviveQueueId={}, can not get biz msg topic is {}, offset is {}, then continue", - queueId, popCheckPoint.getTopic(), msgOffset); - switch (getMessageStatus) { - case MESSAGE_WAS_REMOVING: - case OFFSET_TOO_SMALL: - case NO_MATCHED_LOGIC_QUEUE: - case NO_MESSAGE_IN_QUEUE: - return new Pair<>(msgOffset, true); - default: - return new Pair<>(msgOffset, false); - - } + POP_LOGGER.info("reviveQueueId={}, can not get biz msg, topic:{}, qid:{}, offset:{}, brokerName:{}, info:{}, retry:{}, then continue", + queueId, popCheckPoint.getTopic(), popCheckPoint.getQueueId(), msgOffset, popCheckPoint.getBrokerName(), UtilAll.frontStringAtLeast(rst.getMiddle(), 60), rst.getRight()); + return new Pair<>(msgOffset, !rst.getRight()); // Pair.object2 means OK or not, Triple.right value means needRetry } boolean result = reviveRetry(popCheckPoint, message); return new Pair<>(msgOffset, result); @@ -572,6 +567,13 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { } private void rePutCK(PopCheckPoint oldCK, Pair pair) { + int rePutTimes = oldCK.parseRePutTimes(); + if (rePutTimes >= ckRewriteIntervalsInSeconds.length) { + POP_LOGGER.warn("rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}", oldCK.getTopic(), oldCK.getCId(), + oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime()); + return; + } + PopCheckPoint newCk = new PopCheckPoint(); newCk.setBitMap(0); newCk.setNum((byte) 1); @@ -583,6 +585,11 @@ private void rePutCK(PopCheckPoint oldCK, Pair pair) { newCk.setQueueId(oldCK.getQueueId()); newCk.setBrokerName(oldCK.getBrokerName()); newCk.addDiff(0); + newCk.setRePutTimes(String.valueOf(rePutTimes + 1)); // always increment even if removed from reviveRequestMap + if (oldCK.getReviveTime() <= System.currentTimeMillis()) { + // never expect an ACK matched in the future, we just use it to rewrite CK and try to revive retry message next time + newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[rePutTimes] * 1000); + } MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); brokerController.getMessageStore().putMessage(ckMsg); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 8f89c14ae95..440ebf813bb 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -20,29 +20,43 @@ import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import io.netty.channel.DefaultChannelPromise; +import io.netty.util.concurrent.DefaultEventExecutor; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.auth.config.AuthConfig; import org.apache.rocketmq.broker.out.BrokerOuterAPI; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.BrokerIdentity; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; @@ -56,9 +70,12 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -69,9 +86,11 @@ import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.AdditionalMatchers.or; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(NettyRemotingClient.class) public class BrokerOuterAPITest { @Mock private ChannelHandlerContext handlerContext; @@ -251,4 +270,154 @@ public void testLookupAddressByDomain() throws Exception { }); Assert.assertTrue(result.get()); } + + @Test + public void testPullMessageFromSpecificBrokerAsync_createChannel_null() throws Exception { + NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig())); + PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(null); + BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig()); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(api, mockClient); + + Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertTrue(rst.getMiddle().contains("connect")); + Assert.assertTrue(rst.getRight()); // need retry + } + + @Test + public void testPullMessageFromSpecificBrokerAsync_createChannel_future_notSuccess() throws Exception { + NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig())); + DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor())); + PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise); + BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig()); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(api, mockClient); + + promise.tryFailure(new Throwable()); + Triple rst + = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertTrue(rst.getMiddle().contains("connect")); + Assert.assertTrue(rst.getRight()); // need retry + } + + // skip other future status test + + @Test + public void testPullMessageFromSpecificBrokerAsync_timeout() throws Exception { + Channel channel = Mockito.mock(Channel.class); + when(channel.isActive()).thenReturn(true); + NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig())); + DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor())); + PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise); + when(promise.channel()).thenReturn(channel); + BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig()); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(api, mockClient); + + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong()); + promise.trySuccess(null); + future.completeExceptionally(new RemotingTimeoutException("wait response on the channel timeout")); + Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertTrue(rst.getMiddle().contains("timeout")); + Assert.assertTrue(rst.getRight()); // need retry + } + + @Test + public void testPullMessageFromSpecificBrokerAsync_brokerReturn_pullStatusCode() throws Exception { + Channel channel = Mockito.mock(Channel.class); + when(channel.isActive()).thenReturn(true); + NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig())); + DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor())); + PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise); + when(promise.channel()).thenReturn(channel); + BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig()); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(api, mockClient); + + int[] respCodes = new int[] {ResponseCode.SUCCESS, ResponseCode.PULL_NOT_FOUND, ResponseCode.PULL_RETRY_IMMEDIATELY, ResponseCode.PULL_OFFSET_MOVED}; + PullStatus[] respStatus = new PullStatus[] {PullStatus.FOUND, PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG, PullStatus.OFFSET_ILLEGAL}; + for (int i = 0; i < respCodes.length; i++) { + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong()); + RemotingCommand response = mockPullMessageResponse(respCodes[i]); + ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000, + resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + promise.trySuccess(null); + future.complete(responseFuture); + + Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join(); + Assert.assertEquals(respStatus[i], rst.getLeft().getPullStatus()); + if (ResponseCode.SUCCESS == respCodes[i]) { + Assert.assertEquals(1, rst.getLeft().getMsgFoundList().size()); + } else { + Assert.assertNull(rst.getLeft().getMsgFoundList()); + } + Assert.assertEquals(respStatus[i].name(), rst.getMiddle()); + Assert.assertFalse(rst.getRight()); // no retry + } + } + + @Test + public void testPullMessageFromSpecificBrokerAsync_brokerReturn_allOtherResponseCode() throws Exception { + Channel channel = Mockito.mock(Channel.class); + when(channel.isActive()).thenReturn(true); + NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig())); + DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor())); + PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise); + when(promise.channel()).thenReturn(channel); + BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig()); + Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(api, mockClient); + + CompletableFuture future = new CompletableFuture<>(); + doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong()); + // test one code here, skip others + RemotingCommand response = mockPullMessageResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST); + ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000, + resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + promise.trySuccess(null); + future.complete(responseFuture); + + Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertTrue(rst.getMiddle().contains(ResponseCode.SUBSCRIPTION_NOT_EXIST + "")); + Assert.assertTrue(rst.getRight()); // need retry + } + + private RemotingCommand mockPullMessageResponse(int responseCode) throws Exception { + RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); + response.setCode(responseCode); + if (responseCode == ResponseCode.SUCCESS) { + MessageExt msg = new MessageExt(); + msg.setBody("HW".getBytes()); + msg.setTopic("topic"); + msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000)); + msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000)); + byte[] encode = MessageDecoder.encode(msg, false); + response.setBody(encode); + } + PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader(); + responseHeader.setNextBeginOffset(0L); + responseHeader.setMaxOffset(0L); + responseHeader.setMinOffset(0L); + responseHeader.setOffsetDelta(0L); + responseHeader.setTopicSysFlag(0); + responseHeader.setGroupSysFlag(0); + responseHeader.setSuggestWhichBrokerId(0L); + responseHeader.setForbiddenType(0); + response.makeCustomHeaderToNet(); + return response; + } + } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java index d7bd753d776..7ea06665c3e 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java @@ -17,10 +17,14 @@ package org.apache.rocketmq.broker.failover; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.topic.TopicRouteInfoManager; @@ -28,7 +32,10 @@ import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.PutMessageResult; @@ -38,6 +45,7 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.assertj.core.api.Assertions; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,7 +57,6 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -73,6 +80,12 @@ public class EscapeBridgeTest { @Mock private DefaultMQProducer defaultMQProducer; + @Mock + private TopicRouteInfoManager topicRouteInfoManager; + + @Mock + private BrokerOuterAPI brokerOuterAPI; + private static final String BROKER_NAME = "broker_a"; private static final String TEST_TOPIC = "TEST_TOPIC"; @@ -92,14 +105,10 @@ public void before() throws Exception { when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(getMessageResult)); - TopicRouteInfoManager topicRouteInfoManager = mock(TopicRouteInfoManager.class); when(brokerController.getTopicRouteInfoManager()).thenReturn(topicRouteInfoManager); when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(""); - BrokerOuterAPI brokerOuterAPI = mock(BrokerOuterAPI.class); when(brokerController.getBrokerOuterAPI()).thenReturn(brokerOuterAPI); - when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong())) - .thenReturn(CompletableFuture.completedFuture(new PullResult(PullStatus.FOUND, -1, -1, -1, new ArrayList<>()))); brokerConfig.setEnableSlaveActingMaster(true); brokerConfig.setEnableRemoteEscape(true); @@ -179,6 +188,52 @@ public void getMessageAsyncTest() { Assertions.assertThatCode(() -> escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false)).doesNotThrowAnyException(); } + @Test + public void getMessageAsyncTest_localStore_getMessageAsync_null() { + when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore); + when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(null)); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("getMessageResult is null", rst.getMiddle()); + Assert.assertFalse(rst.getRight()); // no retry + } + + @Test + public void getMessageAsyncTest_localStore_decodeNothing() throws Exception { + when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore); + when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(mockGetMessageResult(0, TEST_TOPIC, null))); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("Can not get msg", rst.getMiddle()); + Assert.assertFalse(rst.getRight()); // no retry + } + + @Test + public void getMessageAsyncTest_localStore_message_found() throws Exception { + when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore); + when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(mockGetMessageResult(2, TEST_TOPIC, "HW".getBytes()))); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNotNull(rst.getLeft()); + Assert.assertEquals(0, rst.getLeft().getQueueOffset()); + Assert.assertTrue(Arrays.equals("HW".getBytes(), rst.getLeft().getBody())); + Assert.assertFalse(rst.getRight()); + } + + @Test + public void getMessageAsyncTest_remoteStore_addressNotFound() throws Exception { + when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(null); + + // just test address not found, since we have complete tests of getMessageFromRemoteAsync() + when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("brokerAddress not found", rst.getMiddle()); + Assert.assertTrue(rst.getRight()); // need retry + } + @Test public void getMessageFromRemoteTest() { Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemote(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException(); @@ -189,6 +244,54 @@ public void getMessageFromRemoteAsyncTest() { Assertions.assertThatCode(() -> escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME)).doesNotThrowAnyException(); } + @Test + public void getMessageFromRemoteAsyncTest_exception_caught() throws Exception { + when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong())) + .thenThrow(new RemotingException("mock remoting exception")); + Triple rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("Get message from remote failed", rst.getMiddle()); + Assert.assertTrue(rst.getRight()); // need retry + } + + @Test + public void getMessageFromRemoteAsyncTest_brokerAddressNotFound() throws Exception { + when(topicRouteInfoManager.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(null); + Triple rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("brokerAddress not found", rst.getMiddle()); + Assert.assertTrue(rst.getRight()); // need retry + } + + @Test + public void getMessageFromRemoteAsyncTest_message_found() throws Exception { + PullResult pullResult = new PullResult(PullStatus.FOUND, 1, 1, 1, Arrays.asList(new MessageExt())); + when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, "", false))); // right value is ignored + Triple rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join(); + Assert.assertNotNull(rst.getLeft()); + Assert.assertTrue(StringUtils.isEmpty(rst.getMiddle())); + Assert.assertFalse(rst.getRight()); // no retry + } + + @Test + public void getMessageFromRemoteAsyncTest_message_notFound() throws Exception { + PullResult pullResult = new PullResult(PullStatus.NO_MATCHED_MSG, 1, 1, 1, null); + when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(pullResult, "no msg", false))); + Triple rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("no msg", rst.getMiddle()); + Assert.assertFalse(rst.getRight()); // no retry + + when(brokerOuterAPI.pullMessageFromSpecificBrokerAsync(anyString(), anyString(), anyString(), anyString(), anyInt(), anyLong(), anyInt(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "other resp code", true))); + rst = escapeBridge.getMessageFromRemoteAsync(TEST_TOPIC, 1, DEFAULT_QUEUE_ID, BROKER_NAME).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("other resp code", rst.getMiddle()); + Assert.assertTrue(rst.getRight()); // need retry + } + @Test public void decodeMsgListTest() { ByteBuffer byteBuffer = ByteBuffer.allocate(10); @@ -199,4 +302,39 @@ public void decodeMsgListTest() { Assertions.assertThatCode(() -> escapeBridge.decodeMsgList(getMessageResult, false)).doesNotThrowAnyException(); } + @Test + public void decodeMsgListTest_messageNotNull() throws Exception { + MessageExt msg = new MessageExt(); + msg.setBody("HW".getBytes()); + msg.setTopic("topic"); + msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000)); + msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000)); + ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false)); + SelectMappedBufferResult result = new SelectMappedBufferResult(0, byteBuffer, 10, new DefaultMappedFile()); + + + getMessageResult.addMessage(result); + getMessageResult.getMessageQueueOffset().add(0L); + List list = escapeBridge.decodeMsgList(getMessageResult, false); // skip deCompressBody test + Assert.assertEquals(1, list.size()); + Assert.assertTrue(Arrays.equals(msg.getBody(), list.get(0).getBody())); + } + + private GetMessageResult mockGetMessageResult(int count, String topic, byte[] body) throws Exception { + GetMessageResult result = new GetMessageResult(); + for (int i = 0; i < count; i++) { + MessageExt msg = new MessageExt(); + msg.setBody(body); + msg.setTopic(topic); + msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000)); + msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000)); + ByteBuffer byteBuffer = ByteBuffer.wrap(MessageDecoder.encode(msg, false)); + SelectMappedBufferResult bufferResult = new SelectMappedBufferResult(0, byteBuffer, body.length, new DefaultMappedFile()); + + result.addMessage(bufferResult); + result.getMessageQueueOffset().add(i + 0L); + } + return result; + } + } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java index 78b76264fef..d7ea97c5502 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java @@ -20,12 +20,17 @@ import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.failover.EscapeBridge; import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.PopAckConstants; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.MessageConst; @@ -36,9 +41,14 @@ import org.apache.rocketmq.common.utils.NetworkUtil; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; import org.apache.rocketmq.store.timer.TimerMessageStore; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -50,19 +60,25 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.times; @RunWith(MockitoJUnitRunner.Silent.class) public class PopReviveServiceTest { - private static final String REVIVE_TOPIC = PopAckConstants.REVIVE_TOPIC + "test"; + private static final String CLUSTER_NAME = "test"; + private static final String REVIVE_TOPIC = PopAckConstants.buildClusterReviveTopic(CLUSTER_NAME); private static final int REVIVE_QUEUE_ID = 0; private static final String GROUP = "group"; private static final String TOPIC = "topic"; private static final SocketAddress STORE_HOST = NetworkUtil.string2SocketAddress("127.0.0.1:8080"); + private static final Long INVISIBLE_TIME = 1000L; @Mock private MessageStore messageStore; @@ -76,6 +92,9 @@ public class PopReviveServiceTest { private SubscriptionGroupManager subscriptionGroupManager; @Mock private BrokerController brokerController; + @Mock + private EscapeBridge escapeBridge; + private PopMessageProcessor popMessageProcessor; private BrokerConfig brokerConfig; private PopReviveService popReviveService; @@ -83,12 +102,14 @@ public class PopReviveServiceTest { @Before public void before() { brokerConfig = new BrokerConfig(); - + brokerConfig.setBrokerClusterName(CLUSTER_NAME); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); when(brokerController.getMessageStore()).thenReturn(messageStore); when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager); + when(brokerController.getEscapeBridge()).thenReturn(escapeBridge); when(messageStore.getTimerMessageStore()).thenReturn(timerMessageStore); when(timerMessageStore.getDequeueBehind()).thenReturn(0L); when(timerMessageStore.getEnqueueBehind()).thenReturn(0L); @@ -96,6 +117,9 @@ public void before() { when(topicConfigManager.selectTopicConfig(anyString())).thenReturn(new TopicConfig()); when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(new SubscriptionGroupConfig()); + popMessageProcessor = new PopMessageProcessor(brokerController); // a real one, not mock + when(brokerController.getPopMessageProcessor()).thenReturn(popMessageProcessor); + popReviveService = spy(new PopReviveService(brokerController, REVIVE_TOPIC, REVIVE_QUEUE_ID)); popReviveService.setShouldRunPopRevive(true); } @@ -204,6 +228,141 @@ public void testSkipLongWaiteAckWithSameAck() throws Throwable { assertEquals(maxReviveOffset, commitOffsetCaptor.getValue().longValue()); } + @Test + public void testReviveMsgFromCk_messageFound_writeRetryOK() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + StringBuilder actualRetryTopic = new StringBuilder(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false))); + when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualRetryTopic.append(msg.getTopic()); + return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)); + }); + + popReviveService.mergeAndRevive(reviveObj); + Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString()); + verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + + @Test + public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + StringBuilder actualRetryTopic = new StringBuilder(); + StringBuilder actualReviveTopic = new StringBuilder(); + AtomicLong actualInvisibleTime = new AtomicLong(0L); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false))); + when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualRetryTopic.append(msg.getTopic()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED)); + }); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualReviveTopic.append(msg.getTopic()); + PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class); + actualInvisibleTime.set(rewriteCK.getReviveTime()); + return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)); + }); + + popReviveService.mergeAndRevive(reviveObj); + Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString()); + Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString()); + Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s + verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + + @Test + public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_end() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + ck.setRePutTimes("17"); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + StringBuilder actualRetryTopic = new StringBuilder(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false))); + when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualRetryTopic.append(msg.getTopic()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED)); + }); + + popReviveService.mergeAndRevive(reviveObj); + Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString()); + verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + + @Test + public void testReviveMsgFromCk_messageNotFound_noRetry() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", false))); + + popReviveService.mergeAndRevive(reviveObj); + verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + + @Test + public void testReviveMsgFromCk_messageNotFound_needRetry() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + StringBuilder actualReviveTopic = new StringBuilder(); + AtomicLong actualInvisibleTime = new AtomicLong(0L); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true))); + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualReviveTopic.append(msg.getTopic()); + PopCheckPoint rewriteCK = JSON.parseObject(msg.getBody(), PopCheckPoint.class); + actualInvisibleTime.set(rewriteCK.getReviveTime()); + return new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)); + }); + + popReviveService.mergeAndRevive(reviveObj); + Assert.assertEquals(REVIVE_TOPIC, actualReviveTopic.toString()); + Assert.assertEquals(INVISIBLE_TIME + 10 * 1000L, actualInvisibleTime.get()); // first interval is 10s + verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + + @Test + public void testReviveMsgFromCk_messageNotFound_needRetry_end() throws Throwable { + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + ck.setRePutTimes("17"); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true))); + + popReviveService.mergeAndRevive(reviveObj); + verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, long reviveOffset) { PopCheckPoint ck = new PopCheckPoint(); ck.setStartOffset(startOffset); @@ -214,7 +373,8 @@ public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, l ck.setNum((byte) 1); ck.setBitMap(0); ck.setReviveOffset(reviveOffset); - ck.setInvisibleTime(1000); + ck.setInvisibleTime(INVISIBLE_TIME); + ck.setBrokerName("broker-a"); return ck; } diff --git a/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java b/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java index e041b66d9c5..38e0a207528 100644 --- a/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java +++ b/store/src/main/java/org/apache/rocketmq/store/pop/PopCheckPoint.java @@ -43,6 +43,8 @@ public class PopCheckPoint implements Comparable { private List queueOffsetDiff; @JSONField(name = "bn") String brokerName; + @JSONField(name = "rp") + String rePutTimes; // ck rePut times public long getReviveOffset() { return reviveOffset; @@ -136,6 +138,14 @@ public void setBrokerName(String brokerName) { this.brokerName = brokerName; } + public String getRePutTimes() { + return rePutTimes; + } + + public void setRePutTimes(String rePutTimes) { + this.rePutTimes = rePutTimes; + } + public void addDiff(int diff) { if (this.queueOffsetDiff == null) { this.queueOffsetDiff = new ArrayList<>(8); @@ -171,10 +181,21 @@ public long ackOffsetByIndex(byte index) { return startOffset + queueOffsetDiff.get(index); } + public int parseRePutTimes() { + if (null == rePutTimes) { + return 0; + } + try { + return Integer.parseInt(rePutTimes); + } catch (Exception e) { + } + return Byte.MAX_VALUE; + } + @Override public String toString() { return "PopCheckPoint [topic=" + topic + ", cid=" + cid + ", queueId=" + queueId + ", startOffset=" + startOffset + ", bitMap=" + bitMap + ", num=" + num + ", reviveTime=" + getReviveTime() - + ", reviveOffset=" + reviveOffset + ", diff=" + queueOffsetDiff + ", brokerName=" + brokerName + "]"; + + ", reviveOffset=" + reviveOffset + ", diff=" + queueOffsetDiff + ", brokerName=" + brokerName + ", rePutTimes=" + rePutTimes + "]"; } @Override From d94d0751a92a43f5368f896418df95cf325a5552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=88=91=E6=98=AF=E7=AE=A1=E5=B0=8F=E4=BA=AE=5FV0x3f?= <42903364+TeFuirnever@users.noreply.github.com> Date: Mon, 5 Aug 2024 09:53:55 +0800 Subject: [PATCH 090/265] [ISSUE #8476] Add test cases for org.apache.rocketmq.common.attribute (#8477) * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8417 Brief Description add test case for AclConfig in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8476 Brief Description add test case for org.apache.rocketmq.common.attribute in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8476 Brief Description add test case for org.apache.rocketmq.common.attribute in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8476 Brief Description add test case for org.apache.rocketmq.common.attribute in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. --- .../common/attribute/AttributeParserTest.java | 72 ++++++++++- .../common/attribute/AttributeTest.java | 43 +++++++ .../common/attribute/AttributeUtilTest.java | 119 ++++++++++++++++++ .../attribute/BooleanAttributeTest.java | 52 ++++++++ .../rocketmq/common/attribute/CQTypeTest.java | 45 +++++++ .../common/attribute/CleanupPolicyTest.java | 36 ++++++ .../common/attribute/EnumAttributeTest.java | 62 +++++++++ .../attribute/LongRangeAttributeTest.java | 65 ++++++++++ .../attribute/TopicMessageTypeTest.java | 63 ++++++++++ 9 files changed, 554 insertions(+), 3 deletions(-) create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java index 12398100bec..a89587354b9 100644 --- a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeParserTest.java @@ -17,18 +17,84 @@ package org.apache.rocketmq.common.attribute; import com.google.common.collect.Maps; -import org.junit.Assert; -import org.junit.Test; - import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.Assert; +import org.junit.Test; import static com.google.common.collect.Maps.newHashMap; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class AttributeParserTest { + + @Test + public void parseToMap_EmptyString_ReturnsEmptyMap() { + String attributesModification = ""; + Map result = AttributeParser.parseToMap(attributesModification); + assertTrue(result.isEmpty()); + } + + @Test + public void parseToMap_NullString_ReturnsEmptyMap() { + String attributesModification = null; + Map result = AttributeParser.parseToMap(attributesModification); + assertTrue(result.isEmpty()); + } + + @Test + public void parseToMap_ValidAttributesModification_ReturnsExpectedMap() { + String attributesModification = "+key1=value1,+key2=value2,-key3,+key4=value4"; + Map result = AttributeParser.parseToMap(attributesModification); + + Map expectedMap = new HashMap<>(); + expectedMap.put("+key1", "value1"); + expectedMap.put("+key2", "value2"); + expectedMap.put("-key3", ""); + expectedMap.put("+key4", "value4"); + + assertEquals(expectedMap, result); + } + + @Test(expected = RuntimeException.class) + public void parseToMap_InvalidAddAttributeFormat_ThrowsRuntimeException() { + String attributesModification = "+key1=value1,key2=value2,-key3,+key4=value4"; + AttributeParser.parseToMap(attributesModification); + } + + @Test(expected = RuntimeException.class) + public void parseToMap_InvalidDeleteAttributeFormat_ThrowsRuntimeException() { + String attributesModification = "+key1=value1,+key2=value2,key3,+key4=value4"; + AttributeParser.parseToMap(attributesModification); + } + + @Test(expected = RuntimeException.class) + public void parseToMap_DuplicateKey_ThrowsRuntimeException() { + String attributesModification = "+key1=value1,+key1=value2"; + AttributeParser.parseToMap(attributesModification); + } + + @Test + public void parseToString_EmptyMap_ReturnsEmptyString() { + Map attributes = new HashMap<>(); + String result = AttributeParser.parseToString(attributes); + assertEquals("", result); + } + + @Test + public void parseToString_ValidAttributes_ReturnsExpectedString() { + Map attributes = new HashMap<>(); + attributes.put("key1", "value1"); + attributes.put("key2", "value2"); + attributes.put("key3", ""); + + String result = AttributeParser.parseToString(attributes); + String expectedString = "key1=value1,key2=value2,key3"; + assertEquals(expectedString, result); + } + @Test public void testParseToMap() { Assert.assertEquals(0, AttributeParser.parseToMap(null).size()); diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java index 39a12b97ef4..9be0f31f06a 100644 --- a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeTest.java @@ -17,12 +17,55 @@ package org.apache.rocketmq.common.attribute; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import static com.google.common.collect.Sets.newHashSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class AttributeTest { + private Attribute attribute; + + @Before + public void setUp() { + attribute = new Attribute("testAttribute", true) { + @Override + public void verify(String value) { + throw new UnsupportedOperationException(); + } + }; + } + + @Test + public void testGetName_ShouldReturnCorrectName() { + assertEquals("testAttribute", attribute.getName()); + } + + @Test + public void testSetName_ShouldSetCorrectName() { + attribute.setName("newTestAttribute"); + assertEquals("newTestAttribute", attribute.getName()); + } + + @Test + public void testIsChangeable_ShouldReturnCorrectChangeableStatus() { + assertTrue(attribute.isChangeable()); + } + + @Test + public void testSetChangeable_ShouldSetCorrectChangeableStatus() { + attribute.setChangeable(false); + assertFalse(attribute.isChangeable()); + } + + @Test(expected = UnsupportedOperationException.class) + public void testVerify_ShouldThrowUnsupportedOperationException() { + attribute.verify("testValue"); + } + @Test public void testEnumAttribute() { EnumAttribute enumAttribute = new EnumAttribute("enum.key", true, newHashSet("enum-1", "enum-2", "enum-3"), "enum-1"); diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java new file mode 100644 index 00000000000..aef46c16680 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/AttributeUtilTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class AttributeUtilTest { + + private Map allAttributes; + private ImmutableMap currentAttributes; + + @Before + public void setUp() { + allAttributes = new HashMap<>(); + allAttributes.put("attr1", new TestAttribute("value1", true, value -> true)); + allAttributes.put("attr2", new TestAttribute("value2", true, value -> true)); + allAttributes.put("attr3", new TestAttribute("value3", true, value -> value.equals("valid"))); + + currentAttributes = ImmutableMap.of("attr1", "value1", "attr2", "value2"); + } + + @Test + public void alterCurrentAttributes_CreateMode_ShouldReturnOnlyAddedAttributes() { + ImmutableMap newAttributes = ImmutableMap.of("+attr1", "new_value1", "+attr2", "value2", "+attr3", "value3"); + + Map result = AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes); + + assertEquals(3, result.size()); + assertTrue(result.containsKey("attr1")); + assertEquals("new_value1", result.get("attr1")); + assertTrue(result.containsKey("attr3")); + assertEquals("value3", result.get("attr3")); + assertTrue(result.containsKey("attr2")); + } + + @Test(expected = RuntimeException.class) + public void alterCurrentAttributes_CreateMode_AddNonAddableAttribute_ShouldThrowException() { + ImmutableMap newAttributes = ImmutableMap.of("attr1", "value1"); + AttributeUtil.alterCurrentAttributes(true, allAttributes, currentAttributes, newAttributes); + } + + @Test + public void alterCurrentAttributes_UpdateMode_ShouldReturnUpdatedAndAddedAttributes() { + ImmutableMap newAttributes = ImmutableMap.of("+attr1", "new_value1", "-attr2", "value2", "+attr3", "value3"); + + Map result = AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes); + + assertEquals(2, result.size()); + assertTrue(result.containsKey("attr1")); + assertEquals("new_value1", result.get("attr1")); + assertTrue(result.containsKey("attr3")); + assertEquals("value3", result.get("attr3")); + assertFalse(result.containsKey("attr2")); + } + + @Test(expected = RuntimeException.class) + public void alterCurrentAttributes_UpdateMode_DeleteNonExistentAttribute_ShouldThrowException() { + ImmutableMap newAttributes = ImmutableMap.of("-attr4", "value4"); + AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes); + } + + @Test(expected = RuntimeException.class) + public void alterCurrentAttributes_UpdateMode_WrongFormatKey_ShouldThrowException() { + ImmutableMap newAttributes = ImmutableMap.of("attr1", "+value1"); + AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes); + } + + @Test(expected = RuntimeException.class) + public void alterCurrentAttributes_UnsupportedKey_ShouldThrowException() { + ImmutableMap newAttributes = ImmutableMap.of("unsupported_attr", "value"); + AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes); + } + + @Test(expected = RuntimeException.class) + public void alterCurrentAttributes_AttemptToUpdateUnchangeableAttribute_ShouldThrowException() { + ImmutableMap newAttributes = ImmutableMap.of("attr2", "new_value2"); + AttributeUtil.alterCurrentAttributes(false, allAttributes, currentAttributes, newAttributes); + } + + private static class TestAttribute extends Attribute { + private final AttributeValidator validator; + + public TestAttribute(String name, boolean changeable, AttributeValidator validator) { + super(name, changeable); + this.validator = validator; + } + + @Override + public void verify(String value) { + validator.validate(value); + } + } + + private interface AttributeValidator { + boolean validate(String value); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java new file mode 100644 index 00000000000..6bed6ffac69 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/BooleanAttributeTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +public class BooleanAttributeTest { + + private BooleanAttribute booleanAttribute; + + @Before + public void setUp() { + booleanAttribute = new BooleanAttribute("testAttribute", true, false); + } + + @Test + public void testVerify_ValidValue_NoExceptionThrown() { + booleanAttribute.verify("true"); + booleanAttribute.verify("false"); + } + + @Test + public void testVerify_InvalidValue_ExceptionThrown() { + assertThrows(RuntimeException.class, () -> booleanAttribute.verify("invalid")); + assertThrows(RuntimeException.class, () -> booleanAttribute.verify("1")); + assertThrows(RuntimeException.class, () -> booleanAttribute.verify("0")); + assertThrows(RuntimeException.class, () -> booleanAttribute.verify("")); + } + + @Test + public void testGetDefaultValue() { + assertFalse(booleanAttribute.getDefaultValue()); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java new file mode 100644 index 00000000000..41aa98ba864 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/CQTypeTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CQTypeTest { + + @Test + public void testValues() { + CQType[] values = CQType.values(); + assertEquals(3, values.length); + assertEquals(CQType.SimpleCQ, values[0]); + assertEquals(CQType.BatchCQ, values[1]); + assertEquals(CQType.RocksDBCQ, values[2]); + } + + @Test + public void testValueOf() { + assertEquals(CQType.SimpleCQ, CQType.valueOf("SimpleCQ")); + assertEquals(CQType.BatchCQ, CQType.valueOf("BatchCQ")); + assertEquals(CQType.RocksDBCQ, CQType.valueOf("RocksDBCQ")); + } + + @Test(expected = IllegalArgumentException.class) + public void testValueOf_InvalidName() { + CQType.valueOf("InvalidCQ"); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java new file mode 100644 index 00000000000..584de2b7e42 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/CleanupPolicyTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CleanupPolicyTest { + + @Test + public void testCleanupPolicy_Delete() { + CleanupPolicy cleanupPolicy = CleanupPolicy.DELETE; + assertEquals("DELETE", cleanupPolicy.toString()); + } + + @Test + public void testCleanupPolicy_Compaction() { + CleanupPolicy cleanupPolicy = CleanupPolicy.COMPACTION; + assertEquals("COMPACTION", cleanupPolicy.toString()); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java new file mode 100644 index 00000000000..637dc302f52 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/EnumAttributeTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import org.junit.Before; +import org.junit.Test; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class EnumAttributeTest { + + private EnumAttribute enumAttribute; + + @Before + public void setUp() { + Set universe = new HashSet<>(); + universe.add("value1"); + universe.add("value2"); + universe.add("value3"); + + enumAttribute = new EnumAttribute("testAttribute", true, universe, "value1"); + } + + @Test + public void verify_ValidValue_NoExceptionThrown() { + enumAttribute.verify("value1"); + enumAttribute.verify("value2"); + enumAttribute.verify("value3"); + } + + @Test + public void verify_InvalidValue_ExceptionThrown() { + RuntimeException exception = assertThrows(RuntimeException.class, () -> { + enumAttribute.verify("invalidValue"); + }); + + assertTrue(exception.getMessage().startsWith("value is not in set:")); + } + + @Test + public void getDefaultValue_ReturnsDefaultValue() { + assertEquals("value1", enumAttribute.getDefaultValue()); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java new file mode 100644 index 00000000000..222f9092d54 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/LongRangeAttributeTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.attribute; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class LongRangeAttributeTest { + + private LongRangeAttribute longRangeAttribute; + + @Before + public void setUp() { + longRangeAttribute = new LongRangeAttribute("testAttribute", true, 0, 100, 50); + } + + @Test + public void verify_ValidValue_NoExceptionThrown() { + longRangeAttribute.verify("50"); + } + + @Test + public void verify_MinValue_NoExceptionThrown() { + longRangeAttribute.verify("0"); + } + + @Test + public void verify_MaxValue_NoExceptionThrown() { + longRangeAttribute.verify("100"); + } + + @Test + public void verify_ValueLessThanMin_ThrowsRuntimeException() { + RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify("-1")); + assertEquals("value is not in range(0, 100)", exception.getMessage()); + } + + @Test + public void verify_ValueGreaterThanMax_ThrowsRuntimeException() { + RuntimeException exception = assertThrows(RuntimeException.class, () -> longRangeAttribute.verify("101")); + assertEquals("value is not in range(0, 100)", exception.getMessage()); + } + + @Test + public void getDefaultValue_ReturnsDefaultValue() { + assertEquals(50, longRangeAttribute.getDefaultValue()); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java index 67525ae8087..0321679ccc0 100644 --- a/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/attribute/TopicMessageTypeTest.java @@ -16,13 +16,76 @@ */ package org.apache.rocketmq.common.attribute; +import com.google.common.collect.Sets; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.apache.rocketmq.common.message.MessageConst; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class TopicMessageTypeTest { + + private Map normalMessageProperty; + private Map transactionMessageProperty; + private Map delayMessageProperty; + private Map fifoMessageProperty; + + @Before + public void setUp() { + normalMessageProperty = new HashMap<>(); + transactionMessageProperty = new HashMap<>(); + delayMessageProperty = new HashMap<>(); + fifoMessageProperty = new HashMap<>(); + + transactionMessageProperty.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + delayMessageProperty.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1"); + fifoMessageProperty.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey"); + } + + @Test + public void testTopicMessageTypeSet() { + Set expectedSet = Sets.newHashSet("UNSPECIFIED", "NORMAL", "FIFO", "DELAY", "TRANSACTION", "MIXED"); + Set actualSet = TopicMessageType.topicMessageTypeSet(); + assertEquals(expectedSet, actualSet); + } + + @Test + public void testParseFromMessageProperty_Normal() { + TopicMessageType actual = TopicMessageType.parseFromMessageProperty(normalMessageProperty); + assertEquals(TopicMessageType.NORMAL, actual); + } + + @Test + public void testParseFromMessageProperty_Transaction() { + TopicMessageType actual = TopicMessageType.parseFromMessageProperty(transactionMessageProperty); + assertEquals(TopicMessageType.TRANSACTION, actual); + } + + @Test + public void testParseFromMessageProperty_Delay() { + TopicMessageType actual = TopicMessageType.parseFromMessageProperty(delayMessageProperty); + assertEquals(TopicMessageType.DELAY, actual); + } + + @Test + public void testParseFromMessageProperty_Fifo() { + TopicMessageType actual = TopicMessageType.parseFromMessageProperty(fifoMessageProperty); + assertEquals(TopicMessageType.FIFO, actual); + } + + @Test + public void testGetMetricsValue() { + for (TopicMessageType type : TopicMessageType.values()) { + String expected = type.getValue().toLowerCase(); + String actual = type.getMetricsValue(); + assertEquals(expected, actual); + } + } + @Test public void testParseFromMessageProperty() { Map properties = new HashMap<>(); From 25b1a0e00eb771214dc123b10ab3edd5fd8c47a4 Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 5 Aug 2024 10:00:50 +0800 Subject: [PATCH 091/265] [ISSUE #8490] Fix getMaxReconsumeTimes calculation error in concurrent consumption mode (#8491) * [ISSUE #8490] Fix getMaxReconsumeTimes calculation error in concurrent consumption mode --- .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index e66a9825f3d..0fef8666cb5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -653,7 +653,7 @@ private PopResult processPopResult(final PopResult popResult, final Subscription Iterator iterator = msgListFilterAgain.iterator(); while (iterator.hasNext()) { MessageExt msg = iterator.next(); - if (msg.getReconsumeTimes() > defaultMQPushConsumer.getMaxReconsumeTimes()) { + if (msg.getReconsumeTimes() > getMaxReconsumeTimes()) { iterator.remove(); log.info("Reconsume times has reached {}, so ack msg={}", msg.getReconsumeTimes(), msg); } From 87fb1f5f7b702fc7c0a2f7f0906c18c3838cedc9 Mon Sep 17 00:00:00 2001 From: Tan Xiang <82364837+TanXiang7o@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:04:10 +0800 Subject: [PATCH 092/265] [ISSUE #8495] Add more test coverage for PeekMessageProcessor (#8498) * [ISSUE #8495]add more test coverage for PeekMessageProcessor * [ISSUE #8495]add more test coverage for PeekMessageProcessor --- .../processor/PeekMessageProcessorTest.java | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java new file mode 100644 index 00000000000..7f8504453ca --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.PeekMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PeekMessageProcessorTest { + + private PeekMessageProcessor peekMessageProcessor; + + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + + @Mock + private ChannelHandlerContext handlerContext; + + @Mock + private MessageStore messageStore; + + @Mock + private SubscriptionGroupManager subscriptionGroupManager; + + @Mock + private ConsumerOffsetManager consumerOffsetManager; + + @Mock + private SubscriptionGroupConfig subscriptionGroupConfig; + + @Mock + private Channel channel; + + private TopicConfigManager topicConfigManager; + + @Before + public void init() { + peekMessageProcessor = new PeekMessageProcessor(brokerController); + when(brokerController.getMessageStore()).thenReturn(messageStore); + topicConfigManager = new TopicConfigManager(brokerController); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager); + when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(subscriptionGroupConfig); + when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(true); + topicConfigManager.getTopicConfigTable().put("topic", new TopicConfig("topic")); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(consumerOffsetManager.queryOffset(anyString(), anyString(), anyInt())).thenReturn(-1L); + when(messageStore.getMinOffsetInQueue(anyString(),anyInt())).thenReturn(0L); + when(handlerContext.channel()).thenReturn(channel); + when(channel.remoteAddress()).thenReturn(new InetSocketAddress("127.0.0.1", 12345)); + } + + @Test + public void testProcessRequest() throws RemotingCommandException { + RemotingCommand request = createPeekMessageRequest("group","topic",0); + GetMessageResult getMessageResult = new GetMessageResult(); + getMessageResult.setStatus(GetMessageStatus.FOUND); + ByteBuffer bb = ByteBuffer.allocate(64); + bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis()); + SelectMappedBufferResult mappedBufferResult1 = new SelectMappedBufferResult(0, bb, 64, null); + for (int i = 0; i < 10;i++) { + getMessageResult.addMessage(mappedBufferResult1); + } + when(messageStore.getMessage(anyString(),anyString(),anyInt(),anyLong(),anyInt(),any())).thenReturn(getMessageResult); + RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testProcessRequest_NoPermission() throws RemotingCommandException { + this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE); + RemotingCommand request = createPeekMessageRequest("group","topic",0); + RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + this.brokerController.getBrokerConfig().setBrokerPermission(PermName.PERM_WRITE | PermName.PERM_READ); + + topicConfigManager.getTopicConfigTable().get("topic").setPerm(PermName.PERM_WRITE); + response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + topicConfigManager.getTopicConfigTable().get("topic").setPerm(PermName.PERM_WRITE | PermName.PERM_READ); + + when(subscriptionGroupConfig.isConsumeEnable()).thenReturn(false); + response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + } + + @Test + public void testProcessRequest_TopicNotExist() throws RemotingCommandException { + RemotingCommand request = createPeekMessageRequest("group1","topic1",0); + RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST); + } + + @Test + public void testProcessRequest_SubscriptionGroupNotExist() throws RemotingCommandException { + when(subscriptionGroupManager.findSubscriptionGroupConfig(anyString())).thenReturn(null); + RemotingCommand request = createPeekMessageRequest("group","topic",0); + RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + } + + @Test + public void testProcessRequest_QueueIdError() throws RemotingCommandException { + RemotingCommand request = createPeekMessageRequest("group","topic",17); + RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + } + + private RemotingCommand createPeekMessageRequest(String group,String topic,int queueId) { + PeekMessageRequestHeader peekMessageRequestHeader = new PeekMessageRequestHeader(); + peekMessageRequestHeader.setConsumerGroup(group); + peekMessageRequestHeader.setTopic(topic); + peekMessageRequestHeader.setMaxMsgNums(10); + peekMessageRequestHeader.setQueueId(queueId); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PEEK_MESSAGE, peekMessageRequestHeader); + request.makeCustomHeaderToNet(); + return request; + } +} From f63a3bae95862e812bed65689368acb79edde20b Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Tue, 6 Aug 2024 17:19:45 +0800 Subject: [PATCH 093/265] [ISSUE #8481] Improve delete and rolling strategy for tiered storage modules (#8493) --- .../tieredstore/MessageStoreConfig.java | 55 ++++++++----------- .../tieredstore/TieredMessageStore.java | 12 +++- .../core/MessageStoreDispatcherImpl.java | 8 ++- .../core/MessageStoreFetcherImpl.java | 23 ++++++-- .../tieredstore/file/FlatCommitLogFile.java | 16 +++++- .../tieredstore/file/FlatFileStore.java | 31 +++++++---- .../tieredstore/index/IndexService.java | 3 + .../tieredstore/index/IndexStoreService.java | 55 ++++++++++++++++--- .../core/MessageStoreDispatcherImplTest.java | 1 + .../file/FlatCommitLogFileTest.java | 1 + .../index/IndexStoreServiceTest.java | 11 +++- 11 files changed, 153 insertions(+), 63 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java index c6e62487309..10667566aa0 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/MessageStoreConfig.java @@ -19,6 +19,7 @@ import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; +import java.time.Duration; public class MessageStoreConfig { @@ -59,6 +60,7 @@ public int getValue() { return value; } + @SuppressWarnings("DuplicatedCode") public static TieredStorageLevel valueOf(int value) { switch (value) { case 1: @@ -91,18 +93,18 @@ public boolean check(TieredStorageLevel targetLevel) { private long tieredStoreConsumeQueueMaxSize = 100 * 1024 * 1024; private int tieredStoreIndexFileMaxHashSlotNum = 5000000; private int tieredStoreIndexFileMaxIndexNum = 5000000 * 4; - // index file will force rolling to next file after idle specified time, default is 3h - private int tieredStoreIndexFileRollingIdleInterval = 3 * 60 * 60 * 1000; + private String tieredMetadataServiceProvider = "org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore"; private String tieredBackendServiceProvider = "org.apache.rocketmq.tieredstore.provider.MemoryFileSegment"; + // file reserved time, default is 72 hour + private boolean tieredStoreDeleteFileEnable = true; private int tieredStoreFileReservedTime = 72; + private long tieredStoreDeleteFileInterval = Duration.ofHours(1).toMillis(); + // time of forcing commitLog to roll to next file, default is 24 hour private int commitLogRollingInterval = 24; - // rolling will only happen if file segment size is larger than commitcp b LogRollingMinimumSize, default is 128M - private int commitLogRollingMinimumSize = 128 * 1024 * 1024; - // default is 100, unit is millisecond - private int maxCommitJitter = 100; + private int commitLogRollingMinimumSize = 16 * 1024 * 1024; private boolean tieredStoreGroupCommit = true; private int tieredStoreGroupCommitTimeout = 30 * 1000; @@ -112,7 +114,6 @@ public boolean check(TieredStorageLevel targetLevel) { private int tieredStoreGroupCommitSize = 4 * 1024 * 1024; // Cached message count larger than this value will suspend append. default is 10000 private int tieredStoreMaxGroupCommitCount = 10000; - private long tieredStoreMaxFallBehindSize = 128 * 1024 * 1024; private boolean readAheadCacheEnable = true; private int readAheadMessageCountThreshold = 4096; @@ -226,14 +227,6 @@ public void setTieredStoreIndexFileMaxIndexNum(int tieredStoreIndexFileMaxIndexN this.tieredStoreIndexFileMaxIndexNum = tieredStoreIndexFileMaxIndexNum; } - public int getTieredStoreIndexFileRollingIdleInterval() { - return tieredStoreIndexFileRollingIdleInterval; - } - - public void setTieredStoreIndexFileRollingIdleInterval(int tieredStoreIndexFileRollingIdleInterval) { - this.tieredStoreIndexFileRollingIdleInterval = tieredStoreIndexFileRollingIdleInterval; - } - public String getTieredMetadataServiceProvider() { return tieredMetadataServiceProvider; } @@ -250,6 +243,14 @@ public void setTieredBackendServiceProvider(String tieredBackendServiceProvider) this.tieredBackendServiceProvider = tieredBackendServiceProvider; } + public boolean isTieredStoreDeleteFileEnable() { + return tieredStoreDeleteFileEnable; + } + + public void setTieredStoreDeleteFileEnable(boolean tieredStoreDeleteFileEnable) { + this.tieredStoreDeleteFileEnable = tieredStoreDeleteFileEnable; + } + public int getTieredStoreFileReservedTime() { return tieredStoreFileReservedTime; } @@ -258,6 +259,14 @@ public void setTieredStoreFileReservedTime(int tieredStoreFileReservedTime) { this.tieredStoreFileReservedTime = tieredStoreFileReservedTime; } + public long getTieredStoreDeleteFileInterval() { + return tieredStoreDeleteFileInterval; + } + + public void setTieredStoreDeleteFileInterval(long tieredStoreDeleteFileInterval) { + this.tieredStoreDeleteFileInterval = tieredStoreDeleteFileInterval; + } + public int getCommitLogRollingInterval() { return commitLogRollingInterval; } @@ -274,14 +283,6 @@ public void setCommitLogRollingMinimumSize(int commitLogRollingMinimumSize) { this.commitLogRollingMinimumSize = commitLogRollingMinimumSize; } - public int getMaxCommitJitter() { - return maxCommitJitter; - } - - public void setMaxCommitJitter(int maxCommitJitter) { - this.maxCommitJitter = maxCommitJitter; - } - public boolean isTieredStoreGroupCommit() { return tieredStoreGroupCommit; } @@ -322,14 +323,6 @@ public void setTieredStoreMaxGroupCommitCount(int tieredStoreMaxGroupCommitCount this.tieredStoreMaxGroupCommitCount = tieredStoreMaxGroupCommitCount; } - public long getTieredStoreMaxFallBehindSize() { - return tieredStoreMaxFallBehindSize; - } - - public void setTieredStoreMaxFallBehindSize(long tieredStoreMaxFallBehindSize) { - this.tieredStoreMaxFallBehindSize = tieredStoreMaxFallBehindSize; - } - public boolean isReadAheadCacheEnable() { return readAheadCacheEnable; } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index 9a25f85a6b8..7b63e16696e 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -104,6 +104,9 @@ public boolean load() { if (result) { indexService.start(); dispatcher.start(); + storeExecutor.commonExecutor.scheduleWithFixedDelay( + flatFileStore::scheduleDeleteExpireFile, storeConfig.getTieredStoreDeleteFileInterval(), + storeConfig.getTieredStoreDeleteFileInterval(), TimeUnit.MILLISECONDS); } return result; } @@ -457,12 +460,12 @@ public synchronized void shutdown() { if (dispatcher != null) { dispatcher.shutdown(); } - if (flatFileStore != null) { - flatFileStore.shutdown(); - } if (indexService != null) { indexService.shutdown(); } + if (flatFileStore != null) { + flatFileStore.shutdown(); + } if (storeExecutor != null) { storeExecutor.shutdown(); } @@ -473,6 +476,9 @@ public void destroy() { if (next != null) { next.destroy(); } + if (indexService != null) { + indexService.destroy(); + } if (flatFileStore != null) { flatFileStore.destroy(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java index 330872ab9cd..ee06700b8b0 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java @@ -138,9 +138,13 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, // If set to max offset here, some written messages may be lost if (!flatFile.isFlatFileInit()) { - currentOffset = Math.max(minOffsetInQueue, - maxOffsetInQueue - storeConfig.getTieredStoreGroupCommitSize()); + currentOffset = defaultStore.getOffsetInQueueByTime( + topic, queueId, System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(2)); + currentOffset = Math.max(currentOffset, minOffsetInQueue); + currentOffset = Math.min(currentOffset, maxOffsetInQueue); flatFile.initOffset(currentOffset); + log.warn("MessageDispatcher#dispatch init, topic={}, queueId={}, offset={}-{}, current={}", + topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset); return CompletableFuture.completedFuture(true); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java index b72ebe86241..7f79dbcd984 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreFetcherImpl.java @@ -39,6 +39,7 @@ import org.apache.rocketmq.tieredstore.file.FlatFileStore; import org.apache.rocketmq.tieredstore.file.FlatMessageFile; import org.apache.rocketmq.tieredstore.index.IndexItem; +import org.apache.rocketmq.tieredstore.index.IndexService; import org.apache.rocketmq.tieredstore.metadata.MetadataStore; import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata; import org.apache.rocketmq.tieredstore.util.MessageFormatUtil; @@ -56,15 +57,24 @@ public class MessageStoreFetcherImpl implements MessageStoreFetcher { private final MetadataStore metadataStore; private final MessageStoreConfig storeConfig; private final TieredMessageStore messageStore; + private final IndexService indexService; private final FlatFileStore flatFileStore; private final long memoryMaxSize; private final Cache fetcherCache; public MessageStoreFetcherImpl(TieredMessageStore messageStore) { - this.storeConfig = messageStore.getStoreConfig(); + this(messageStore, messageStore.getStoreConfig(), + messageStore.getFlatFileStore(), messageStore.getIndexService()); + } + + public MessageStoreFetcherImpl(TieredMessageStore messageStore, MessageStoreConfig storeConfig, + FlatFileStore flatFileStore, IndexService indexService) { + + this.storeConfig = storeConfig; this.brokerName = storeConfig.getBrokerName(); - this.flatFileStore = messageStore.getFlatFileStore(); + this.flatFileStore = flatFileStore; this.messageStore = messageStore; + this.indexService = indexService; this.metadataStore = flatFileStore.getMetadataStore(); this.memoryMaxSize = (long) (Runtime.getRuntime().maxMemory() * storeConfig.getReadAheadCacheSizeThresholdRate()); @@ -192,7 +202,11 @@ public CompletableFuture getMessageFromCacheAsync( log.debug("MessageFetcher cache miss, group={}, topic={}, queueId={}, offset={}, maxCount={}, lag={}", group, mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, result.getMaxOffset() - result.getNextBeginOffset()); - return fetchMessageThenPutToCache(flatFile, queueOffset, storeConfig.getReadAheadMessageCountThreshold()) + // To optimize the performance of pop consumption + // Pop revive will cause a large number of random reads, + // so the amount of pre-fetch message num needs to be reduced. + int fetchSize = maxCount == 1 ? 32 : storeConfig.getReadAheadMessageCountThreshold(); + return fetchMessageThenPutToCache(flatFile, queueOffset, fetchSize) .thenApply(maxOffset -> getMessageFromCache(flatFile, queueOffset, maxCount, messageFilter)); } @@ -414,8 +428,7 @@ public CompletableFuture queryMessageAsync( return CompletableFuture.completedFuture(new QueryMessageResult()); } - CompletableFuture> future = - messageStore.getIndexService().queryAsync(topic, key, maxCount, begin, end); + CompletableFuture> future = indexService.queryAsync(topic, key, maxCount, begin, end); return future.thenCompose(indexItemList -> { List> futureList = new ArrayList<>(maxCount); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java index 6ac0939571f..16c05204759 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFile.java @@ -19,6 +19,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.tieredstore.common.FileSegmentType; +import org.apache.rocketmq.tieredstore.provider.FileSegment; import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory; import org.apache.rocketmq.tieredstore.util.MessageFormatUtil; @@ -33,10 +34,19 @@ public FlatCommitLogFile(FileSegmentFactory fileSegmentFactory, String filePath) this.initOffset(0L); } + /** + * Two rules are set here: + * 1. Single file must be saved for more than one day as default. + * 2. Single file must reach the minimum size before switching. + * When calculating storage space, due to the limitation of condition 2, + * the actual usage of storage space may be slightly higher than expected. + */ public boolean tryRollingFile(long interval) { - long timestamp = this.getFileToWrite().getMinTimestamp(); - if (timestamp != Long.MAX_VALUE && - timestamp + interval < System.currentTimeMillis()) { + FileSegment fileSegment = this.getFileToWrite(); + long timestamp = fileSegment.getMinTimestamp(); + if (timestamp != Long.MAX_VALUE && timestamp + interval < System.currentTimeMillis() && + fileSegment.getAppendPosition() >= + fileSegmentFactory.getStoreConfig().getCommitLogRollingMinimumSize()) { this.rollingNewFile(this.getAppendOffset()); return true; } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java index f782d099def..70ba2178010 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java @@ -59,16 +59,6 @@ public boolean load() { try { this.flatFileConcurrentMap.clear(); this.recover(); - this.executor.commonExecutor.scheduleWithFixedDelay(() -> { - for (FlatMessageFile flatFile : deepCopyFlatFileToList()) { - long expiredTimeStamp = System.currentTimeMillis() - - TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours()); - flatFile.destroyExpiredFile(expiredTimeStamp); - if (flatFile.consumeQueue.fileSegmentTable.isEmpty()) { - this.destroyFile(flatFile.getMessageQueue()); - } - } - }, 60, 60, TimeUnit.SECONDS); log.info("FlatFileStore recover finished, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); } catch (Exception e) { long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); @@ -113,6 +103,27 @@ public CompletableFuture recoverAsync(TopicMetadata topicMetadata) { }, executor.bufferCommitExecutor); } + public void scheduleDeleteExpireFile() { + if (!storeConfig.isTieredStoreDeleteFileEnable()) { + return; + } + Stopwatch stopwatch = Stopwatch.createStarted(); + ImmutableList fileList = this.deepCopyFlatFileToList(); + for (FlatMessageFile flatFile : fileList) { + flatFile.getFileLock().lock(); + try { + flatFile.destroyExpiredFile(System.currentTimeMillis() - + TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())); + } catch (Exception e) { + log.error("FlatFileStore delete expire file error", e); + } finally { + flatFile.getFileLock().unlock(); + } + } + log.info("FlatFileStore schedule delete expired file, count={}, cost={}ms", + fileList.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + public MetadataStore getMetadataStore() { return metadataStore; } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java index 70c36c88042..a4ea7e78a85 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexService.java @@ -52,6 +52,9 @@ AppendResult putKey( */ CompletableFuture> queryAsync(String topic, String key, int maxCount, long beginTime, long endTime); + default void forceUpload() { + } + /** * Shutdown the index service. */ diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java index 9e53d97b98c..020b9f3b068 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java @@ -42,6 +42,8 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.common.AppendResult; +import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode; +import org.apache.rocketmq.tieredstore.exception.TieredStoreException; import org.apache.rocketmq.tieredstore.file.FlatAppendFile; import org.apache.rocketmq.tieredstore.file.FlatFileFactory; import org.apache.rocketmq.tieredstore.provider.FileSegment; @@ -66,18 +68,29 @@ public class IndexStoreService extends ServiceThread implements IndexService { private final AtomicLong compactTimestamp; private final String filePath; private final FlatFileFactory fileAllocator; + private final boolean autoCreateNewFile; - private IndexFile currentWriteFile; - private FlatAppendFile flatAppendFile; + private volatile IndexFile currentWriteFile; + private volatile FlatAppendFile flatAppendFile; public IndexStoreService(FlatFileFactory flatFileFactory, String filePath) { + this(flatFileFactory, filePath, true); + } + + public IndexStoreService(FlatFileFactory flatFileFactory, String filePath, boolean autoCreateNewFile) { this.storeConfig = flatFileFactory.getStoreConfig(); this.filePath = filePath; this.fileAllocator = flatFileFactory; this.timeStoreTable = new ConcurrentSkipListMap<>(); this.compactTimestamp = new AtomicLong(0L); this.readWriteLock = new ReentrantReadWriteLock(); + this.autoCreateNewFile = autoCreateNewFile; + } + + @Override + public void start() { this.recover(); + super.start(); } private void doConvertOldFormatFile(String filePath) { @@ -131,12 +144,14 @@ private void recover() { } } - if (this.timeStoreTable.isEmpty()) { + if (this.autoCreateNewFile && this.timeStoreTable.isEmpty()) { this.createNewIndexFile(System.currentTimeMillis()); } - this.currentWriteFile = this.timeStoreTable.lastEntry().getValue(); - this.setCompactTimestamp(this.timeStoreTable.firstKey() - 1); + if (!this.timeStoreTable.isEmpty()) { + this.currentWriteFile = this.timeStoreTable.lastEntry().getValue(); + this.setCompactTimestamp(this.timeStoreTable.firstKey() - 1); + } // recover remote this.flatAppendFile = fileAllocator.createFlatFileForIndexFile(filePath); @@ -206,7 +221,7 @@ public AppendResult putKey( log.error("IndexStoreService put key three times return error, topic: {}, topicId: {}, " + "queueId: {}, keySize: {}, timestamp: {}", topic, topicId, queueId, keySet.size(), timestamp); - return AppendResult.UNKNOWN_ERROR; + return AppendResult.SUCCESS; } @Override @@ -252,6 +267,30 @@ public CompletableFuture> queryAsync( return future; } + @Override + public void forceUpload() { + try { + readWriteLock.writeLock().lock(); + if (this.currentWriteFile == null) { + log.warn("IndexStoreService no need force upload current write file"); + return; + } + // note: current file has been shutdown before + IndexStoreFile lastFile = new IndexStoreFile(storeConfig, currentWriteFile.getTimestamp()); + if (this.doCompactThenUploadFile(lastFile)) { + this.setCompactTimestamp(lastFile.getTimestamp()); + } else { + throw new TieredStoreException( + TieredStoreErrorCode.UNKNOWN, "IndexStoreService force compact current file error"); + } + } catch (Exception e) { + log.error("IndexStoreService force upload error", e); + throw new RuntimeException(e); + } finally { + readWriteLock.writeLock().lock(); + } + } + public boolean doCompactThenUploadFile(IndexFile indexFile) { if (IndexFile.IndexStatusEnum.UPLOAD.equals(indexFile.getFileStatus())) { log.error("IndexStoreService file status not correct, so skip, timestamp: {}, status: {}", @@ -359,6 +398,9 @@ public void shutdown() { for (Map.Entry entry : timeStoreTable.entrySet()) { entry.getValue().shutdown(); } + if (!autoCreateNewFile) { + this.forceUpload(); + } this.timeStoreTable.clear(); } catch (Exception e) { log.error("IndexStoreService shutdown error", e); @@ -373,7 +415,6 @@ public void run() { long expireTimestamp = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); this.destroyExpiredFile(expireTimestamp); - IndexFile indexFile = this.getNextSealedFile(); if (indexFile != null) { if (this.doCompactThenUploadFile(indexFile)) { diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java index 8ac7e068a76..92e989e596f 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImplTest.java @@ -99,6 +99,7 @@ public void dispatchFromCommitLogTest() throws Exception { messageStore = Mockito.mock(TieredMessageStore.class); IndexService indexService = new IndexStoreService(new FlatFileFactory(metadataStore, storeConfig), storePath); + indexService.start(); Mockito.when(messageStore.getDefaultStore()).thenReturn(defaultStore); Mockito.when(messageStore.getStoreConfig()).thenReturn(storeConfig); Mockito.when(messageStore.getStoreExecutor()).thenReturn(executor); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java index 1e912690b2f..0fbf5a6a843 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/FlatCommitLogFileTest.java @@ -77,6 +77,7 @@ public void tryRollingFileTest() throws InterruptedException { byteBuffer.putLong(MessageFormatUtil.QUEUE_OFFSET_POSITION, i); Assert.assertEquals(AppendResult.SUCCESS, flatFile.append(byteBuffer, i)); TimeUnit.MILLISECONDS.sleep(2); + storeConfig.setCommitLogRollingMinimumSize(byteBuffer.remaining()); Assert.assertTrue(flatFile.tryRollingFile(1)); } Assert.assertEquals(4, flatFile.fileSegmentTable.size()); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java index ec55a028bb9..fb563f7c6c2 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java @@ -93,6 +93,7 @@ public void shutdown() { @Test public void basicServiceTest() throws InterruptedException { indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); for (int i = 0; i < 50; i++) { Assert.assertEquals(AppendResult.SUCCESS, indexService.putKey( TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, i * 100, MESSAGE_SIZE, System.currentTimeMillis())); @@ -105,6 +106,7 @@ public void basicServiceTest() throws InterruptedException { @Test public void doConvertOldFormatTest() throws IOException { indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); long timestamp = indexService.getTimeStoreTable().firstKey(); Assert.assertEquals(AppendResult.SUCCESS, indexService.putKey( TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, MESSAGE_OFFSET, MESSAGE_SIZE, timestamp)); @@ -116,6 +118,7 @@ public void doConvertOldFormatTest() throws IOException { mappedFile.shutdown(10 * 1000); indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); ConcurrentSkipListMap timeStoreTable = indexService.getTimeStoreTable(); Assert.assertEquals(1, timeStoreTable.size()); Assert.assertEquals(Long.valueOf(timestamp), timeStoreTable.firstKey()); @@ -129,6 +132,7 @@ public void concurrentPutTest() throws InterruptedException { storeConfig.setTieredStoreIndexFileMaxHashSlotNum(500); storeConfig.setTieredStoreIndexFileMaxIndexNum(2000); indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); long timestamp = System.currentTimeMillis(); // first item is invalid @@ -205,6 +209,7 @@ public void runServiceTest() throws InterruptedException { @Test public void restartServiceTest() throws InterruptedException { indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); for (int i = 0; i < 20; i++) { AppendResult result = indexService.putKey( TOPIC_NAME, TOPIC_ID, QUEUE_ID, Collections.singleton(String.valueOf(i)), @@ -214,10 +219,10 @@ public void restartServiceTest() throws InterruptedException { } long timestamp = indexService.getTimeStoreTable().firstKey(); indexService.shutdown(); - indexService = new IndexStoreService(fileAllocator, filePath); - Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue()); + indexService = new IndexStoreService(fileAllocator, filePath); indexService.start(); + Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue()); await().atMost(Duration.ofMinutes(1)).pollInterval(Duration.ofSeconds(1)).until(() -> { ArrayList files = new ArrayList<>(indexService.getTimeStoreTable().values()); return IndexFile.IndexStatusEnum.UPLOAD.equals(files.get(0).getFileStatus()); @@ -225,6 +230,7 @@ public void restartServiceTest() throws InterruptedException { indexService.shutdown(); indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue()); Assert.assertEquals(2, indexService.getTimeStoreTable().size()); Assert.assertEquals(IndexFile.IndexStatusEnum.UPLOAD, @@ -235,6 +241,7 @@ public void restartServiceTest() throws InterruptedException { public void queryFromFileTest() throws InterruptedException, ExecutionException { long timestamp = System.currentTimeMillis(); indexService = new IndexStoreService(fileAllocator, filePath); + indexService.start(); // three files, echo contains 19 items for (int i = 0; i < 3; i++) { From cba8a64b08bf1b30ad2db6619b06d758d8cb3955 Mon Sep 17 00:00:00 2001 From: ziiyee <137812228+ziiyee@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:05:19 +0800 Subject: [PATCH 094/265] [ISSUE #8486]Add more test coverage for BrokerMetricsManager (#8487) * Add more test case for BrokerMetricsManager. Includes: - check the topic is retry or dlq topic - check the group is system group or not - check the topic and the group belongs to system or not * Add more test case for BrokerMetricsManager. Check topic message type by request header. * Add more test case for BrokerMetricsManager. * Add more test case for BrokerMetricsManager. --- .../metrics/BrokerMetricsManagerTest.java | 277 +++++++++++++++++- 1 file changed, 275 insertions(+), 2 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java index 11f7ae8215a..9264eb4b56b 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManagerTest.java @@ -20,8 +20,25 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.metrics.MetricsExporterType; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.Test; +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + import static org.assertj.core.api.Assertions.assertThat; public class BrokerMetricsManagerTest { @@ -29,7 +46,7 @@ public class BrokerMetricsManagerTest { @Test public void testNewAttributesBuilder() { Attributes attributes = BrokerMetricsManager.newAttributesBuilder().put("a", "b") - .build(); + .build(); assertThat(attributes.get(AttributeKey.stringKey("a"))).isEqualTo("b"); } @@ -37,6 +54,7 @@ public void testNewAttributesBuilder() { public void testCustomizedAttributesBuilder() { BrokerMetricsManager.attributesBuilderSupplier = () -> new AttributesBuilder() { private AttributesBuilder attributesBuilder = Attributes.builder(); + @Override public Attributes build() { return attributesBuilder.put("customized", "value").build(); @@ -61,8 +79,263 @@ public AttributesBuilder putAll(Attributes attributes) { } }; Attributes attributes = BrokerMetricsManager.newAttributesBuilder().put("a", "b") - .build(); + .build(); assertThat(attributes.get(AttributeKey.stringKey("a"))).isEqualTo("b"); assertThat(attributes.get(AttributeKey.stringKey("customized"))).isEqualTo("value"); } + + + @Test + public void testIsRetryOrDlqTopicWithRetryTopic() { + String topic = MixAll.RETRY_GROUP_TOPIC_PREFIX + "TestTopic"; + boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic); + assertThat(result).isTrue(); + } + + @Test + public void testIsRetryOrDlqTopicWithDlqTopic() { + String topic = MixAll.DLQ_GROUP_TOPIC_PREFIX + "TestTopic"; + boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic); + assertThat(result).isTrue(); + } + + @Test + public void testIsRetryOrDlqTopicWithNonRetryOrDlqTopic() { + String topic = "NormalTopic"; + boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic); + assertThat(result).isFalse(); + } + + @Test + public void testIsRetryOrDlqTopicWithEmptyTopic() { + String topic = ""; + boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic); + assertThat(result).isFalse(); + } + + @Test + public void testIsRetryOrDlqTopicWithNullTopic() { + String topic = null; + boolean result = BrokerMetricsManager.isRetryOrDlqTopic(topic); + assertThat(result).isFalse(); + } + + @Test + public void testIsSystemGroup_SystemGroup_ReturnsTrue() { + String group = "FooGroup"; + String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group; + boolean result = BrokerMetricsManager.isSystemGroup(systemGroup); + assertThat(result).isTrue(); + } + + @Test + public void testIsSystemGroup_NonSystemGroup_ReturnsFalse() { + String group = "FooGroup"; + boolean result = BrokerMetricsManager.isSystemGroup(group); + assertThat(result).isFalse(); + } + + @Test + public void testIsSystemGroup_EmptyGroup_ReturnsFalse() { + String group = ""; + boolean result = BrokerMetricsManager.isSystemGroup(group); + assertThat(result).isFalse(); + } + + @Test + public void testIsSystemGroup_NullGroup_ReturnsFalse() { + String group = null; + boolean result = BrokerMetricsManager.isSystemGroup(group); + assertThat(result).isFalse(); + } + + @Test + public void testIsSystem_SystemTopicOrSystemGroup_ReturnsTrue() { + String topic = "FooTopic"; + String group = "FooGroup"; + String systemTopic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC; + String systemGroup = MixAll.CID_RMQ_SYS_PREFIX + group; + + boolean resultTopic = BrokerMetricsManager.isSystem(systemTopic, group); + assertThat(resultTopic).isTrue(); + + boolean resultGroup = BrokerMetricsManager.isSystem(topic, systemGroup); + assertThat(resultGroup).isTrue(); + } + + @Test + public void testIsSystem_NonSystemTopicAndGroup_ReturnsFalse() { + String topic = "FooTopic"; + String group = "FooGroup"; + boolean result = BrokerMetricsManager.isSystem(topic, group); + assertThat(result).isFalse(); + } + + @Test + public void testIsSystem_EmptyTopicAndGroup_ReturnsFalse() { + String topic = ""; + String group = ""; + boolean result = BrokerMetricsManager.isSystem(topic, group); + assertThat(result).isFalse(); + } + + @Test + public void testGetMessageTypeAsNormal() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProperties(""); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.NORMAL).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsTransaction() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.TRANSACTION).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsFifo() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.FIFO).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsDelayLevel() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.DELAY).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsDeliverMS() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TIMER_DELIVER_MS, "10"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.DELAY).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsDelaySEC() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TIMER_DELAY_SEC, "1"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.DELAY).isEqualTo(result); + } + + @Test + public void testGetMessageTypeAsDelayMS() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TIMER_DELAY_MS, "10"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.DELAY).isEqualTo(result); + } + + @Test + public void testGetMessageTypeWithUnknownProperty() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put("unknownProperty", "unknownValue"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.NORMAL).isEqualTo(result); + } + + @Test + public void testGetMessageTypeWithMultipleProperties() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "1"); + map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.FIFO).isEqualTo(result); + } + + @Test + public void testGetMessageTypeWithTransactionFlagButOtherPropertiesPresent() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + Map map = new HashMap<>(); + map.put(MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + map.put(MessageConst.PROPERTY_SHARDING_KEY, "shardingKey"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + + TopicMessageType result = BrokerMetricsManager.getMessageType(requestHeader); + assertThat(TopicMessageType.TRANSACTION).isEqualTo(result); + } + + @Test + public void testGetMessageTypeWithEmptyProperties() { + TopicMessageType result = BrokerMetricsManager.getMessageType(new SendMessageRequestHeader()); + assertThat(TopicMessageType.NORMAL).isEqualTo(result); + } + + @Test + public void testCreateMetricsManager() { + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + String storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + + UUID.randomUUID(); + messageStoreConfig.setStorePathRootDir(storePathRootDir); + BrokerConfig brokerConfig = new BrokerConfig(); + + NettyServerConfig nettyServerConfig = new NettyServerConfig(); + nettyServerConfig.setListenPort(0); + + BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, + new NettyClientConfig(), messageStoreConfig); + + BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController); + + assertThat(metricsManager.getBrokerMeter()).isNull(); + } + + @Test + public void testCreateMetricsManagerLogType() throws CloneNotSupportedException { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setMetricsExporterType(MetricsExporterType.LOG); + brokerConfig.setMetricsLabel("label1:value1;label2:value2"); + brokerConfig.setMetricsOtelCardinalityLimit(1); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + String storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + + UUID.randomUUID(); + messageStoreConfig.setStorePathRootDir(storePathRootDir); + + NettyServerConfig nettyServerConfig = new NettyServerConfig(); + nettyServerConfig.setListenPort(0); + + BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, + new NettyClientConfig(), messageStoreConfig); + brokerController.initialize(); + + BrokerMetricsManager metricsManager = new BrokerMetricsManager(brokerController); + + assertThat(metricsManager.getBrokerMeter()).isNotNull(); + } } \ No newline at end of file From f7c27bba2a113c1020f2c86d9505de797f753772 Mon Sep 17 00:00:00 2001 From: guning <56331831+StudentGu@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:45:15 +0800 Subject: [PATCH 095/265] [ISSUE #8500] Add more test coverage for RocksDBLmqConsumerOffsetManager (#8502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ```新增RocksDBLmqConsumerOffsetManagerTest.java测试用例``` * ```新增RocksDBLmqConsumerOffsetManagerTest.java测试用例``` * ```新增RocksDBLmqConsumerOffsetManagerTest.java测试用例``` --- .../RocksDBLmqConsumerOffsetManagerTest.java | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java new file mode 100644 index 00000000000..ea6528546dc --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.offset; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +public class RocksDBLmqConsumerOffsetManagerTest { + private static final String LMQ_GROUP = MixAll.LMQ_PREFIX + "FooBarGroup"; + private static final String NON_LMQ_GROUP = "nonLmqGroup"; + private static final String TOPIC = "FooBarTopic"; + private static final int QUEUE_ID = 0; + private static final long OFFSET = 12345; + + private BrokerController brokerController; + + private RocksDBLmqConsumerOffsetManager offsetManager; + + @Before + public void setUp() { + brokerController = Mockito.mock(BrokerController.class); + when(brokerController.getMessageStoreConfig()).thenReturn(Mockito.mock(MessageStoreConfig.class)); + when(brokerController.getBrokerConfig()).thenReturn(Mockito.mock(BrokerConfig.class)); + offsetManager = new RocksDBLmqConsumerOffsetManager(brokerController); + } + + @Test + public void testQueryOffsetForLmq() { + // Setup + offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); + // Execute + long actualOffset = offsetManager.queryOffset(LMQ_GROUP, TOPIC, QUEUE_ID); + // Verify + assertEquals("Offset should match the expected value.", OFFSET, actualOffset); + } + + @Test + public void testQueryOffsetForNonLmq() { + long actualOffset = offsetManager.queryOffset(NON_LMQ_GROUP, TOPIC, QUEUE_ID); + // Verify + assertEquals("Offset should not be null.", -1, actualOffset); + } + + + @Test + public void testQueryOffsetForLmqGroupWithExistingOffset() { + offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); + + // Act + Map actualOffsets = offsetManager.queryOffset(LMQ_GROUP, TOPIC); + + // Assert + assertNotNull(actualOffsets); + assertEquals(1, actualOffsets.size()); + assertEquals(OFFSET, (long) actualOffsets.get(0)); + } + + @Test + public void testQueryOffsetForLmqGroupWithoutExistingOffset() { + // Act + Map actualOffsets = offsetManager.queryOffset(LMQ_GROUP, "nonExistingTopic"); + + // Assert + assertNotNull(actualOffsets); + assertTrue("The map should be empty for non-existing offsets", actualOffsets.isEmpty()); + } + + @Test + public void testQueryOffsetForNonLmqGroup() { + when(brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep()).thenReturn(1L); + // Arrange + Map mockOffsets = new HashMap<>(); + mockOffsets.put(QUEUE_ID, OFFSET); + + offsetManager.commitOffset("clientHost", NON_LMQ_GROUP, TOPIC, QUEUE_ID, OFFSET); + + // Act + Map actualOffsets = offsetManager.queryOffset(NON_LMQ_GROUP, TOPIC); + + // Assert + assertNotNull(actualOffsets); + assertEquals("Offsets should match the mocked return value for non-LMQ groups", mockOffsets, actualOffsets); + } + + @Test + public void testCommitOffsetForLmq() { + // Execute + offsetManager.commitOffset("clientHost", LMQ_GROUP, TOPIC, QUEUE_ID, OFFSET); + // Verify + Long expectedOffset = offsetManager.getLmqOffsetTable().get(getKey()); + assertEquals("Offset should be updated correctly.", OFFSET, expectedOffset.longValue()); + } + + @Test + public void testEncode() { + offsetManager.setLmqOffsetTable(new ConcurrentHashMap<>(512)); + offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); + String encodedData = offsetManager.encode(); + assertTrue(encodedData.contains(String.valueOf(OFFSET))); + } + + private String getKey() { + return TOPIC + "@" + LMQ_GROUP; + } +} From f4deb0e226c30ddcb8e9947a2144436dfdb8fbf0 Mon Sep 17 00:00:00 2001 From: yx9o Date: Thu, 8 Aug 2024 10:46:37 +0800 Subject: [PATCH 096/265] [ISSUE #8496] Add more test coverage for ConsumeMessagePopOrderlyService (#8497) * [ISSUE #8496] Add more test coverage for ConsumeMessagePopOrderlyService * Update --- .../ConsumeMessagePopOrderlyServiceTest.java | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java new file mode 100644 index 00000000000..257783ecb48 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessagePopOrderlyServiceTest.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.stat.ConsumerStatsManager; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.body.CMResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ConsumeMessagePopOrderlyServiceTest { + + @Mock + private DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + + @Mock + private MessageListenerOrderly messageListener; + + @Mock + private DefaultMQPushConsumer defaultMQPushConsumer; + + @Mock + private ConsumerStatsManager consumerStatsManager; + + @Mock + private RebalanceImpl rebalanceImpl; + + private ConsumeMessagePopOrderlyService popService; + + private final String defaultGroup = "defaultGroup"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultTopic = "defaultTopic"; + + @Before + public void init() throws Exception { + when(defaultMQPushConsumer.getConsumerGroup()).thenReturn(defaultGroup); + when(defaultMQPushConsumer.getConsumeThreadMin()).thenReturn(1); + when(defaultMQPushConsumer.getConsumeThreadMax()).thenReturn(3); + when(defaultMQPushConsumerImpl.getDefaultMQPushConsumer()).thenReturn(defaultMQPushConsumer); + when(defaultMQPushConsumerImpl.getRebalanceImpl()).thenReturn(rebalanceImpl); + when(defaultMQPushConsumerImpl.getConsumerStatsManager()).thenReturn(consumerStatsManager); + MQClientInstance mQClientFactory = mock(MQClientInstance.class); + DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class); + when(mQClientFactory.getDefaultMQProducer()).thenReturn(defaultMQProducer); + when(defaultMQPushConsumerImpl.getmQClientFactory()).thenReturn(mQClientFactory); + popService = new ConsumeMessagePopOrderlyService(defaultMQPushConsumerImpl, messageListener); + } + + @Test + public void testShutdown() throws IllegalAccessException { + popService.shutdown(3000L); + Field scheduledExecutorServiceField = FieldUtils.getDeclaredField(popService.getClass(), "scheduledExecutorService", true); + Field consumeExecutorField = FieldUtils.getDeclaredField(popService.getClass(), "consumeExecutor", true); + ScheduledExecutorService scheduledExecutorService = (ScheduledExecutorService) scheduledExecutorServiceField.get(popService); + ThreadPoolExecutor consumeExecutor = (ThreadPoolExecutor) consumeExecutorField.get(popService); + assertTrue(scheduledExecutorService.isShutdown()); + assertTrue(scheduledExecutorService.isTerminated()); + assertTrue(consumeExecutor.isShutdown()); + assertTrue(consumeExecutor.isTerminated()); + } + + @Test + public void testUnlockAllMessageQueues() { + popService.unlockAllMessageQueues(); + verify(rebalanceImpl, times(1)).unlockAll(eq(false)); + } + + @Test + public void testUpdateCorePoolSize() { + popService.updateCorePoolSize(2); + popService.incCorePoolSize(); + popService.decCorePoolSize(); + assertEquals(2, popService.getCorePoolSize()); + } + + @Test + public void testConsumeMessageDirectly() { + when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUCCESS); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_SUCCESS, actual.getConsumeResult()); + assertTrue(actual.isOrder()); + } + + @Test + public void testConsumeMessageDirectlyWithCommit() { + when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.COMMIT); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_COMMIT, actual.getConsumeResult()); + assertTrue(actual.isOrder()); + } + + @Test + public void testConsumeMessageDirectlyWithRollback() { + when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.ROLLBACK); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_ROLLBACK, actual.getConsumeResult()); + assertTrue(actual.isOrder()); + } + + @Test + public void testConsumeMessageDirectlyWithCrLater() { + when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenReturn(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_LATER, actual.getConsumeResult()); + } + + @Test + public void testConsumeMessageDirectlyWithCrReturnNull() { + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_RETURN_NULL, actual.getConsumeResult()); + } + + @Test + public void testConsumeMessageDirectlyWithCrThrowException() { + when(messageListener.consumeMessage(any(), any(ConsumeOrderlyContext.class))).thenThrow(new RuntimeException("exception")); + ConsumeMessageDirectlyResult actual = popService.consumeMessageDirectly(createMessageExt(), defaultBroker); + assertEquals(CMResult.CR_THROW_EXCEPTION, actual.getConsumeResult()); + } + + @Test + public void testSubmitConsumeRequest() { + assertThrows(UnsupportedOperationException.class, () -> { + List msgs = mock(List.class); + ProcessQueue processQueue = mock(ProcessQueue.class); + MessageQueue messageQueue = mock(MessageQueue.class); + popService.submitConsumeRequest(msgs, processQueue, messageQueue, false); + }); + } + + @Test + public void testSubmitPopConsumeRequest() throws IllegalAccessException { + List msgs = Collections.singletonList(createMessageExt()); + PopProcessQueue processQueue = mock(PopProcessQueue.class); + MessageQueue messageQueue = mock(MessageQueue.class); + ThreadPoolExecutor consumeExecutor = mock(ThreadPoolExecutor.class); + FieldUtils.writeDeclaredField(popService, "consumeExecutor", consumeExecutor, true); + popService.submitPopConsumeRequest(msgs, processQueue, messageQueue); + verify(consumeExecutor, times(1)).submit(any(Runnable.class)); + } + + @Test + public void testLockMQPeriodically() { + popService.lockMQPeriodically(); + verify(defaultMQPushConsumerImpl, times(1)).getRebalanceImpl(); + verify(rebalanceImpl, times(1)).lockAll(); + } + + @Test + public void testGetConsumerStatsManager() { + ConsumerStatsManager actual = popService.getConsumerStatsManager(); + assertNotNull(actual); + assertEquals(consumerStatsManager, actual); + } + + @Test + public void testSendMessageBack() { + assertTrue(popService.sendMessageBack(createMessageExt())); + } + + @Test + public void testProcessConsumeResult() { + ConsumeOrderlyContext context = mock(ConsumeOrderlyContext.class); + ConsumeMessagePopOrderlyService.ConsumeRequest consumeRequest = mock(ConsumeMessagePopOrderlyService.ConsumeRequest.class); + assertTrue(popService.processConsumeResult(Collections.singletonList(createMessageExt()), ConsumeOrderlyStatus.SUCCESS, context, consumeRequest)); + } + + @Test + public void testResetNamespace() { + when(defaultMQPushConsumer.getNamespace()).thenReturn("defaultNamespace"); + List msgs = Collections.singletonList(createMessageExt()); + popService.resetNamespace(msgs); + assertEquals(defaultTopic, msgs.get(0).getTopic()); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + long curTime = System.currentTimeMillis(); + result.setBornTimestamp(curTime - 1000); + result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } +} From 8167608334c901bd85482904ed203a26ac8950e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=88=91=E6=98=AF=E7=AE=A1=E5=B0=8F=E4=BA=AE=5FV0x3f?= <42903364+TeFuirnever@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:47:14 +0800 Subject: [PATCH 097/265] [ISSUE #8503] Add test cases for org.apache.rocketmq.common.chain/coldstr/compression/consumer (#8504) * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8417 Brief Description add test case for AclConfig in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8476 Brief Description add test case for org.apache.rocketmq.common.attribute in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes add test case for AclConfig in commom module Fixes #8476 Brief Description add test case for org.apache.rocketmq.common.attribute in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes Add test cases for org.apache.rocketmq.common.chain/coldstr/compression/consumer Fixes apache#8503 Brief Description add test case for org.apache.rocketmq.common.chain/coldstr/compression/consumer in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes Add test cases for org.apache.rocketmq.common.chain/coldstr/compression/consumer Fixes #8503 Brief Description add test case for org.apache.rocketmq.common.chain/coldstr/compression/consumer in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. * Which Issue(s) This PR Fixes Add test cases for org.apache.rocketmq.common.chain/coldstr/compression/consumer Fixes #8503 Brief Description add test case for org.apache.rocketmq.common.chain/coldstr/compression/consumer in commom module by using tongyi tools. How Did You Test This Change? run test case successfull. --- .../common/chain/HandlerChainTest.java | 65 +++++++++++ .../common/coldctr/AccAndTimeStampTest.java | 70 ++++++++++++ .../compression/CompressionTypeTest.java | 60 ++++++++++ .../compression/CompressorFactoryTest.java | 42 +++++++ .../common/compression/Lz4CompressorTest.java | 53 +++++++++ .../compression/ZlibCompressorTest.java | 53 +++++++++ .../compression/ZstdCompressorTest.java | 78 +++++++++++++ .../common/consumer/ReceiptHandleTest.java | 103 ++++++++++++++++++ 8 files changed, 524 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java b/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java new file mode 100644 index 00000000000..3a8499ebad2 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/chain/HandlerChainTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.chain; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class HandlerChainTest { + + private HandlerChain handlerChain; + private Handler handler1; + private Handler handler2; + + @Before + public void setUp() { + handlerChain = HandlerChain.create(); + handler1 = (t, chain) -> "Handler1"; + handler2 = (t, chain) -> null; + } + + @Test + public void testHandle_withEmptyChain() { + handlerChain.addNext(handler1); + handlerChain.handle(1); + assertNull("Expected null since the handler chain is empty", handlerChain.handle(2)); + } + + @Test + public void testHandle_withNonEmptyChain() { + handlerChain.addNext(handler1); + + String result = handlerChain.handle(1); + + assertEquals("Handler1", result); + } + + @Test + public void testHandle_withMultipleHandlers() { + handlerChain.addNext(handler1); + handlerChain.addNext(handler2); + + String result1 = handlerChain.handle(1); + String result2 = handlerChain.handle(2); + + assertEquals("Handler1", result1); + assertNull("Expected null since there are no more handlers", result2); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java b/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java new file mode 100644 index 00000000000..01bb4ae3701 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/coldctr/AccAndTimeStampTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.coldctr; + +import java.util.concurrent.atomic.AtomicLong; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class AccAndTimeStampTest { + + private AccAndTimeStamp accAndTimeStamp; + + @Before + public void setUp() { + accAndTimeStamp = new AccAndTimeStamp(new AtomicLong()); + } + + @Test + public void testInitialValues() { + assertEquals("Cold accumulator should be initialized to 0", 0, accAndTimeStamp.getColdAcc().get()); + assertTrue("Last cold read time should be initialized to current time", accAndTimeStamp.getLastColdReadTimeMills() >= System.currentTimeMillis() - 1000); + assertTrue("Create time should be initialized to current time", accAndTimeStamp.getCreateTimeMills() >= System.currentTimeMillis() - 1000); + } + + @Test + public void testSetColdAcc() { + AtomicLong newColdAcc = new AtomicLong(100L); + accAndTimeStamp.setColdAcc(newColdAcc); + assertEquals("Cold accumulator should be set to new value", newColdAcc, accAndTimeStamp.getColdAcc()); + } + + @Test + public void testSetLastColdReadTimeMills() { + long newLastColdReadTimeMills = System.currentTimeMillis() + 1000; + accAndTimeStamp.setLastColdReadTimeMills(newLastColdReadTimeMills); + assertEquals("Last cold read time should be set to new value", newLastColdReadTimeMills, accAndTimeStamp.getLastColdReadTimeMills().longValue()); + } + + @Test + public void testSetCreateTimeMills() { + long newCreateTimeMills = System.currentTimeMillis() + 2000; + accAndTimeStamp.setCreateTimeMills(newCreateTimeMills); + assertEquals("Create time should be set to new value", newCreateTimeMills, accAndTimeStamp.getCreateTimeMills().longValue()); + } + + @Test + public void testToStringContainsCorrectInformation() { + String toStringOutput = accAndTimeStamp.toString(); + assertTrue("ToString should contain cold accumulator value", toStringOutput.contains("coldAcc=" + accAndTimeStamp.getColdAcc())); + assertTrue("ToString should contain last cold read time", toStringOutput.contains("lastColdReadTimeMills=" + accAndTimeStamp.getLastColdReadTimeMills())); + assertTrue("ToString should contain create time", toStringOutput.contains("createTimeMills=" + accAndTimeStamp.getCreateTimeMills())); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java new file mode 100644 index 00000000000..e0ec18fd44b --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.compression; + +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class CompressionTypeTest { + + @Test + public void testCompressionTypeValues() { + assertEquals(1, CompressionType.LZ4.getValue(), "LZ4 value should be 1"); + assertEquals(2, CompressionType.ZSTD.getValue(), "ZSTD value should be 2"); + assertEquals(3, CompressionType.ZLIB.getValue(), "ZLIB value should be 3"); + } + + @Test + public void testCompressionTypeOf() { + assertEquals(CompressionType.LZ4, CompressionType.of("LZ4"), "CompressionType.of(LZ4) should return LZ4"); + assertEquals(CompressionType.ZSTD, CompressionType.of("ZSTD"), "CompressionType.of(ZSTD) should return ZSTD"); + assertEquals(CompressionType.ZLIB, CompressionType.of("ZLIB"), "CompressionType.of(ZLIB) should return ZLIB"); + + assertThrows(RuntimeException.class, () -> CompressionType.of("UNKNOWN"), "Unsupported compression type should throw RuntimeException"); + } + + @Test + public void testCompressionTypeFindByValue() { + assertEquals(CompressionType.LZ4, CompressionType.findByValue(1), "CompressionType.findByValue(1) should return LZ4"); + assertEquals(CompressionType.ZSTD, CompressionType.findByValue(2), "CompressionType.findByValue(2) should return ZSTD"); + assertEquals(CompressionType.ZLIB, CompressionType.findByValue(3), "CompressionType.findByValue(3) should return ZLIB"); + + assertEquals(CompressionType.ZLIB, CompressionType.findByValue(0), "CompressionType.findByValue(0) should return ZLIB for backward compatibility"); + + assertThrows(RuntimeException.class, () -> CompressionType.findByValue(99), "Invalid value should throw RuntimeException"); + } + + @Test + public void testCompressionFlag() { + assertEquals(MessageSysFlag.COMPRESSION_LZ4_TYPE, CompressionType.LZ4.getCompressionFlag(), "LZ4 compression flag is incorrect"); + assertEquals(MessageSysFlag.COMPRESSION_ZSTD_TYPE, CompressionType.ZSTD.getCompressionFlag(), "ZSTD compression flag is incorrect"); + assertEquals(MessageSysFlag.COMPRESSION_ZLIB_TYPE, CompressionType.ZLIB.getCompressionFlag(), "ZLIB compression flag is incorrect"); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java new file mode 100644 index 00000000000..e150fb2f7aa --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/compression/CompressorFactoryTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.compression; + +import org.junit.Assert; +import org.junit.Test; + +public class CompressorFactoryTest { + + @Test + public void testGetCompressor_ReturnsNonNull() { + for (CompressionType type : CompressionType.values()) { + Compressor compressor = CompressorFactory.getCompressor(type); + Assert.assertNotNull("Compressor should not be null for type " + type, compressor); + } + } + + @Test + public void testGetCompressor_ReturnsCorrectType() { + for (CompressionType type : CompressionType.values()) { + Compressor compressor = CompressorFactory.getCompressor(type); + Assert.assertTrue("Compressor type mismatch for " + type, + compressor instanceof Lz4Compressor && type == CompressionType.LZ4 || + compressor instanceof ZstdCompressor && type == CompressionType.ZSTD || + compressor instanceof ZlibCompressor && type == CompressionType.ZLIB); + } + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java new file mode 100644 index 00000000000..ca59025c133 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/compression/Lz4CompressorTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.compression; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import org.junit.Test; + +public class Lz4CompressorTest { + + private static final String TEST_STRING = "The quick brown fox jumps over the lazy dog"; + + @Test + public void testCompressAndDecompress() throws Exception { + byte[] originalData = TEST_STRING.getBytes(); + Compressor compressor = new Lz4Compressor(); + byte[] compressedData = compressor.compress(originalData, 1); + assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length); + + byte[] decompressedData = compressor.decompress(compressedData); + assertArrayEquals("Decompressed data should match original data", originalData, decompressedData); + } + + @Test + public void testCompressWithIOException() throws Exception { + byte[] originalData = new byte[] {1, 2, 3}; + Compressor compressor = new Lz4Compressor(); + compressor.compress(originalData, 1); + } + + @Test(expected = IOException.class) + public void testDecompressWithIOException() throws Exception { + byte[] compressedData = new byte[] {1, 2, 3}; + Compressor compressor = new Lz4Compressor(); + compressor.decompress(compressedData); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java new file mode 100644 index 00000000000..f46ac7c6691 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/compression/ZlibCompressorTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.compression; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import org.junit.Test; + +public class ZlibCompressorTest { + + private static final String TEST_STRING = "The quick brown fox jumps over the lazy dog"; + + @Test + public void testCompressionAndDecompression() throws Exception { + byte[] originalData = TEST_STRING.getBytes(); + ZlibCompressor compressor = new ZlibCompressor(); + byte[] compressedData = compressor.compress(originalData, 0); + assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length); + + byte[] decompressedData = compressor.decompress(compressedData); + assertArrayEquals("Decompressed data should match original", originalData, decompressedData); + } + + @Test + public void testCompressionFailureWithInvalidData() throws Exception { + byte[] originalData = new byte[] {0, 1, 2, 3, 4}; + ZlibCompressor compressor = new ZlibCompressor(); + compressor.compress(originalData, 0); + } + + @Test(expected = IOException.class) + public void testDecompressionFailureWithInvalidData() throws Exception { + byte[] compressedData = new byte[] {0, 1, 2, 3, 4}; + ZlibCompressor compressor = new ZlibCompressor(); + compressor.decompress(compressedData); // Invalid compressed data + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java new file mode 100644 index 00000000000..574e1281811 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/compression/ZstdCompressorTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.compression; + +import java.io.IOException; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +public class ZstdCompressorTest { + + @Test + public void testCompressAndDecompress() throws IOException { + byte[] originalData = "RocketMQ is awesome!".getBytes(); + ZstdCompressor compressor = new ZstdCompressor(); + byte[] compressedData = compressor.compress(originalData, 1); + assertTrue("Compressed data should be bigger than original", compressedData.length > originalData.length); + + byte[] decompressedData = compressor.decompress(compressedData); + assertArrayEquals("Decompressed data should match original data", originalData, decompressedData); + } + + @Test + public void testCompressWithInvalidData() throws IOException { + byte[] invalidData = new byte[] {-1, -1, -1, -1}; + ZstdCompressor compressor = new ZstdCompressor(); + compressor.compress(invalidData, 1); + } + + @Test(expected = IOException.class) + public void testDecompressWithInvalidData() throws IOException { + byte[] invalidData = new byte[] {-1, -1, -1, -1}; + ZstdCompressor compressor = new ZstdCompressor(); + compressor.decompress(invalidData); + } + + @Test + public void testCompressAndDecompressEmptyString() throws IOException { + byte[] originalData = "".getBytes(); + ZstdCompressor compressor = new ZstdCompressor(); + byte[] compressedData = compressor.compress(originalData, 1); + assertTrue("Compressed data for empty string should not be empty", compressedData.length > 0); + + byte[] decompressedData = compressor.decompress(compressedData); + assertArrayEquals("Decompressed data for empty string should match original", originalData, decompressedData); + } + + @Test + public void testCompressAndDecompressLargeData() throws IOException { + StringBuilder largeStringBuilder = new StringBuilder(); + for (int i = 0; i < 10000; i++) { + largeStringBuilder.append("RocketMQ is awesome! "); + } + byte[] originalData = largeStringBuilder.toString().getBytes(); + + ZstdCompressor compressor = new ZstdCompressor(); + byte[] compressedData = compressor.compress(originalData, 1); + assertTrue("Compressed data for large data should be smaller than original", compressedData.length < originalData.length); + + byte[] decompressedData = compressor.decompress(compressedData); + assertArrayEquals("Decompressed data for large data should match original", originalData, decompressedData); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java b/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java new file mode 100644 index 00000000000..54741817e12 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/consumer/ReceiptHandleTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.consumer; + +import org.apache.rocketmq.common.KeyBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ReceiptHandleTest { + + @Test + public void testEncodeAndDecode() { + long startOffset = 1000L; + long retrieveTime = System.currentTimeMillis(); + long invisibleTime = 1000L; + int reviveQueueId = 1; + String topicType = "NORMAL"; + String brokerName = "BrokerA"; + int queueId = 2; + long offset = 2000L; + long commitLogOffset = 3000L; + ReceiptHandle receiptHandle = ReceiptHandle.builder() + .startOffset(startOffset) + .retrieveTime(retrieveTime) + .invisibleTime(invisibleTime) + .reviveQueueId(reviveQueueId) + .topicType(topicType) + .brokerName(brokerName) + .queueId(queueId) + .offset(offset) + .commitLogOffset(commitLogOffset) + .build(); + + String encoded = receiptHandle.encode(); + ReceiptHandle decoded = ReceiptHandle.decode(encoded); + + assertEquals(receiptHandle.getStartOffset(), decoded.getStartOffset()); + assertEquals(receiptHandle.getRetrieveTime(), decoded.getRetrieveTime()); + assertEquals(receiptHandle.getInvisibleTime(), decoded.getInvisibleTime()); + assertEquals(receiptHandle.getReviveQueueId(), decoded.getReviveQueueId()); + assertEquals(receiptHandle.getTopicType(), decoded.getTopicType()); + assertEquals(receiptHandle.getBrokerName(), decoded.getBrokerName()); + assertEquals(receiptHandle.getQueueId(), decoded.getQueueId()); + assertEquals(receiptHandle.getOffset(), decoded.getOffset()); + assertEquals(receiptHandle.getCommitLogOffset(), decoded.getCommitLogOffset()); + } + + @Test(expected = IllegalArgumentException.class) + public void testDecodeWithInvalidString() { + String invalidReceiptHandle = "invalid_data"; + + ReceiptHandle.decode(invalidReceiptHandle); + } + + @Test + public void testIsExpired() { + long startOffset = 1000L; + long retrieveTime = System.currentTimeMillis(); + long invisibleTime = 1000L; + int reviveQueueId = 1; + String topicType = "NORMAL"; + String brokerName = "BrokerA"; + int queueId = 2; + long offset = 2000L; + long commitLogOffset = 3000L; + long pastTime = System.currentTimeMillis() - 1000L; + ReceiptHandle receiptHandle = new ReceiptHandle(startOffset, retrieveTime, invisibleTime, pastTime, reviveQueueId, topicType, brokerName, queueId, offset, commitLogOffset, ""); + + boolean isExpired = receiptHandle.isExpired(); + + assertTrue(isExpired); + } + + @Test + public void testGetRealTopic() { + // Arrange + String topic = "TestTopic"; + String groupName = "TestGroup"; + ReceiptHandle receiptHandle = ReceiptHandle.builder() + .topicType(ReceiptHandle.RETRY_TOPIC) + .build(); + + String realTopic = receiptHandle.getRealTopic(topic, groupName); + + assertEquals(KeyBuilder.buildPopRetryTopicV1(topic, groupName), realTopic); + } +} From 606cefeb31187f38babea0b786a216a0bc6df238 Mon Sep 17 00:00:00 2001 From: yx9o Date: Sat, 10 Aug 2024 21:07:36 +0800 Subject: [PATCH 098/265] [ISSUE #8514] Fix bazel-compile (ubuntu-latest) ci run failure (#8515) * [ISSUE #8514] Fix bazel-compile (ubuntu-latest) ci run failure * Update * Update --- .../compression/CompressionTypeTest.java | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java index e0ec18fd44b..f9586bd2da8 100644 --- a/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/compression/CompressionTypeTest.java @@ -17,44 +17,41 @@ package org.apache.rocketmq.common.compression; import org.apache.rocketmq.common.sysflag.MessageSysFlag; -import org.junit.jupiter.api.Test; +import org.junit.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class CompressionTypeTest { @Test public void testCompressionTypeValues() { - assertEquals(1, CompressionType.LZ4.getValue(), "LZ4 value should be 1"); - assertEquals(2, CompressionType.ZSTD.getValue(), "ZSTD value should be 2"); - assertEquals(3, CompressionType.ZLIB.getValue(), "ZLIB value should be 3"); + assertEquals(1, CompressionType.LZ4.getValue()); + assertEquals(2, CompressionType.ZSTD.getValue()); + assertEquals(3, CompressionType.ZLIB.getValue()); } @Test public void testCompressionTypeOf() { - assertEquals(CompressionType.LZ4, CompressionType.of("LZ4"), "CompressionType.of(LZ4) should return LZ4"); - assertEquals(CompressionType.ZSTD, CompressionType.of("ZSTD"), "CompressionType.of(ZSTD) should return ZSTD"); - assertEquals(CompressionType.ZLIB, CompressionType.of("ZLIB"), "CompressionType.of(ZLIB) should return ZLIB"); - - assertThrows(RuntimeException.class, () -> CompressionType.of("UNKNOWN"), "Unsupported compression type should throw RuntimeException"); + assertEquals(CompressionType.LZ4, CompressionType.of("LZ4")); + assertEquals(CompressionType.ZSTD, CompressionType.of("ZSTD")); + assertEquals(CompressionType.ZLIB, CompressionType.of("ZLIB")); + assertThrows(RuntimeException.class, () -> CompressionType.of("UNKNOWN")); } @Test public void testCompressionTypeFindByValue() { - assertEquals(CompressionType.LZ4, CompressionType.findByValue(1), "CompressionType.findByValue(1) should return LZ4"); - assertEquals(CompressionType.ZSTD, CompressionType.findByValue(2), "CompressionType.findByValue(2) should return ZSTD"); - assertEquals(CompressionType.ZLIB, CompressionType.findByValue(3), "CompressionType.findByValue(3) should return ZLIB"); - - assertEquals(CompressionType.ZLIB, CompressionType.findByValue(0), "CompressionType.findByValue(0) should return ZLIB for backward compatibility"); - - assertThrows(RuntimeException.class, () -> CompressionType.findByValue(99), "Invalid value should throw RuntimeException"); + assertEquals(CompressionType.LZ4, CompressionType.findByValue(1)); + assertEquals(CompressionType.ZSTD, CompressionType.findByValue(2)); + assertEquals(CompressionType.ZLIB, CompressionType.findByValue(3)); + assertEquals(CompressionType.ZLIB, CompressionType.findByValue(0)); + assertThrows(RuntimeException.class, () -> CompressionType.findByValue(99)); } @Test public void testCompressionFlag() { - assertEquals(MessageSysFlag.COMPRESSION_LZ4_TYPE, CompressionType.LZ4.getCompressionFlag(), "LZ4 compression flag is incorrect"); - assertEquals(MessageSysFlag.COMPRESSION_ZSTD_TYPE, CompressionType.ZSTD.getCompressionFlag(), "ZSTD compression flag is incorrect"); - assertEquals(MessageSysFlag.COMPRESSION_ZLIB_TYPE, CompressionType.ZLIB.getCompressionFlag(), "ZLIB compression flag is incorrect"); + assertEquals(MessageSysFlag.COMPRESSION_LZ4_TYPE, CompressionType.LZ4.getCompressionFlag()); + assertEquals(MessageSysFlag.COMPRESSION_ZSTD_TYPE, CompressionType.ZSTD.getCompressionFlag()); + assertEquals(MessageSysFlag.COMPRESSION_ZLIB_TYPE, CompressionType.ZLIB.getCompressionFlag()); } } From aed259b4c9ca7a407a600e5079fd51192aa33751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Mon, 12 Aug 2024 17:29:01 +0800 Subject: [PATCH 099/265] [ISSUE #8510] Fix CI Failure in Test E2E Golang Job of PUSH-CI and PR-E2E-TEST (#8520) * Modify the golang e2e test script * Update cmd * Revert changes to push-ci.yml * Update go version --- .github/workflows/pr-e2e-test.yml | 2 ++ .github/workflows/push-ci.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index 9082b6b2227..f9bb3bde75a 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -187,6 +187,8 @@ jobs: test-cmd: | cd ../common && mvn -Prelease -DskipTests clean package -U cd ../rocketmq-admintools && source bin/env.sh + wget https://go.dev/dl/go1.22.6.linux-amd64.tar.gz && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.6.linux-amd64.tar.gz cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v job-id: 0 - name: Publish Test Report diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index b679d56d2f0..2fe62dbeb06 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -192,6 +192,8 @@ jobs: test-cmd: | cd ../common && mvn -Prelease -DskipTests clean package -U cd ../rocketmq-admintools && source bin/env.sh + wget https://go.dev/dl/go1.22.6.linux-amd64.tar.gz && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.6.linux-amd64.tar.gz cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v job-id: 0 - name: Publish Test Report From fa0cd49df20223269fa659f940539b40064359a3 Mon Sep 17 00:00:00 2001 From: dinglei Date: Tue, 13 Aug 2024 11:33:12 +0800 Subject: [PATCH 100/265] [ISSUE #8499]Modify batch send delay time to 3000ms in unit test. (#8522) * modify batch send delay time to 3000ms in unit test. * close unittest in macOS platform. * close unittest in macOS platform checking. * close broker regest timeout unittest in macOS platform checking. * close broker regest timeout unittest in macOS platform checking. * modify batch send delay time to 3000ms in unit test. --- .../java/org/apache/rocketmq/broker/BrokerOuterAPITest.java | 3 +++ .../rocketmq/client/producer/ProduceAccumulatorTest.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 440ebf813bb..1d12acd4a98 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -136,6 +136,9 @@ public void test_needRegister_normal() throws Exception { @Test public void test_needRegister_timeout() throws Exception { + if (MixAll.isMac()) { + return; + } init(); brokerOuterAPI.start(); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java index 7074fae243d..8e76238d47f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/ProduceAccumulatorTest.java @@ -106,6 +106,7 @@ public void testProduceAccumulator_sync() throws MQBrokerException, RemotingExce final MockMQProducer mockMQProducer = new MockMQProducer(); final ProduceAccumulator produceAccumulator = new ProduceAccumulator("test"); + produceAccumulator.batchMaxDelayMs(3000); produceAccumulator.start(); List messages = new ArrayList(); @@ -134,7 +135,7 @@ public void run() { } }).start(); } - assertThat(countDownLatch.await(3000L, TimeUnit.MILLISECONDS)).isTrue(); + assertThat(countDownLatch.await(5000L, TimeUnit.MILLISECONDS)).isTrue(); assertThat(mockMQProducer.beSendMessage instanceof MessageBatch).isTrue(); MessageBatch messageBatch1 = (MessageBatch) mockMQProducer.beSendMessage; From 7f5bf5f89336998d0bfc0b0f468ccbb4ddce90d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=98=9F=E7=81=BF?= <37405937+qianye1001@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:21:11 +0800 Subject: [PATCH 101/265] [ISSUE #8517] Fix client send UNREGISTER_CLIENT request twice may cause proxy NPE (#8528) --- .../activity/ClientManagerActivity.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java index c671593a34b..05d8e5fbe13 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/ClientManagerActivity.java @@ -125,22 +125,30 @@ protected RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCo final String producerGroup = requestHeader.getProducerGroup(); if (producerGroup != null) { RemotingChannel channel = this.remotingChannelManager.removeProducerChannel(context, producerGroup, ctx.channel()); - ClientChannelInfo clientChannelInfo = new ClientChannelInfo( - channel, - requestHeader.getClientID(), - request.getLanguage(), - request.getVersion()); - this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo); + if (channel != null) { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + channel, + requestHeader.getClientID(), + request.getLanguage(), + request.getVersion()); + this.messagingProcessor.unRegisterProducer(context, producerGroup, clientChannelInfo); + } else { + log.warn("unregister producer failed, channel not exist, may has been removed, producerGroup={}, channel={}", producerGroup, ctx.channel()); + } } final String consumerGroup = requestHeader.getConsumerGroup(); if (consumerGroup != null) { RemotingChannel channel = this.remotingChannelManager.removeConsumerChannel(context, consumerGroup, ctx.channel()); - ClientChannelInfo clientChannelInfo = new ClientChannelInfo( - channel, - requestHeader.getClientID(), - request.getLanguage(), - request.getVersion()); - this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo); + if (channel != null) { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + channel, + requestHeader.getClientID(), + request.getLanguage(), + request.getVersion()); + this.messagingProcessor.unRegisterConsumer(context, consumerGroup, clientChannelInfo); + } else { + log.warn("unregister consumer failed, channel not exist, may has been removed, consumerGroup={}, channel={}", consumerGroup, ctx.channel()); + } } response.setCode(ResponseCode.SUCCESS); response.setRemark(""); From d3d7b280a1a5d9d2f69af0aa74affe71c9232513 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 14 Aug 2024 10:18:19 +0800 Subject: [PATCH 102/265] [ISSUE #8517] Add more test coverage for PullAPIWrapper (#8518) * [ISSUE #8517] Add more test coverage for PullAPIWrapper * Update * Update --- .../impl/consumer/PullAPIWrapperTest.java | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java new file mode 100644 index 00000000000..2ffa8f4f149 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapperTest.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.PopCallback; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.FilterMessageContext; +import org.apache.rocketmq.client.hook.FilterMessageHook; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PullAPIWrapperTest { + + @Mock + private MQClientInstance mQClientFactory; + + @Mock + private MQClientAPIImpl mqClientAPIImpl; + + private PullAPIWrapper pullAPIWrapper; + + private final String defaultGroup = "defaultGroup"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBrokerAddr = "127.0.0.1:10911"; + + private final long defaultTimeout = 3000L; + + @Before + public void init() throws Exception { + ClientConfig clientConfig = mock(ClientConfig.class); + when(mQClientFactory.getClientConfig()).thenReturn(clientConfig); + MQClientAPIImpl mqClientAPIImpl = mock(MQClientAPIImpl.class); + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl); + when(mQClientFactory.getTopicRouteTable()).thenReturn(createTopicRouteTable()); + FindBrokerResult findBrokerResult = mock(FindBrokerResult.class); + when(findBrokerResult.getBrokerAddr()).thenReturn(defaultBrokerAddr); + when(mQClientFactory.findBrokerAddressInSubscribe(any(), anyLong(), anyBoolean())).thenReturn(findBrokerResult); + pullAPIWrapper = new PullAPIWrapper(mQClientFactory, defaultGroup, false); + ArrayList filterMessageHookList = new ArrayList<>(); + filterMessageHookList.add(mock(FilterMessageHook.class)); + FieldUtils.writeDeclaredField(pullAPIWrapper, "filterMessageHookList", filterMessageHookList, true); + } + + @Test + public void testProcessPullResult() throws Exception { + PullResultExt pullResult = mock(PullResultExt.class); + when(pullResult.getPullStatus()).thenReturn(PullStatus.FOUND); + when(pullResult.getMessageBinary()).thenReturn(MessageDecoder.encode(createMessageExt(), false)); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + PullResult actual = pullAPIWrapper.processPullResult(createMessageQueue(), pullResult, subscriptionData); + assertNotNull(actual); + assertEquals(0, actual.getNextBeginOffset()); + assertEquals(0, actual.getMsgFoundList().size()); + } + + @Test + public void testExecuteHook() throws IllegalAccessException { + FilterMessageContext filterMessageContext = mock(FilterMessageContext.class); + ArrayList filterMessageHookList = new ArrayList<>(); + FilterMessageHook filterMessageHook = mock(FilterMessageHook.class); + filterMessageHookList.add(filterMessageHook); + FieldUtils.writeDeclaredField(pullAPIWrapper, "filterMessageHookList", filterMessageHookList, true); + pullAPIWrapper.executeHook(filterMessageContext); + verify(filterMessageHook, times(1)).filterMessage(any(FilterMessageContext.class)); + } + + @Test + public void testPullKernelImpl() throws Exception { + PullCallback pullCallback = mock(PullCallback.class); + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl); + PullResult actual = pullAPIWrapper.pullKernelImpl(createMessageQueue(), + "", + "", + 1L, + 1L, + 1, + 1, + PullSysFlag.buildSysFlag(false, false, false, true), + 1L, + System.currentTimeMillis(), + defaultTimeout, CommunicationMode.ASYNC, pullCallback); + assertNull(actual); + verify(mqClientAPIImpl, times(1)).pullMessage(eq(defaultBroker), + any(PullMessageRequestHeader.class), + eq(defaultTimeout), + any(CommunicationMode.class), + any(PullCallback.class)); + } + + @Test + public void testSetConnectBrokerByUser() { + pullAPIWrapper.setConnectBrokerByUser(true); + assertTrue(pullAPIWrapper.isConnectBrokerByUser()); + } + + @Test + public void testRandomNum() { + int randomNum = pullAPIWrapper.randomNum(); + assertTrue(randomNum > 0); + } + + @Test + public void testSetDefaultBrokerId() { + pullAPIWrapper.setDefaultBrokerId(MixAll.MASTER_ID); + assertEquals(MixAll.MASTER_ID, pullAPIWrapper.getDefaultBrokerId()); + } + + @Test + public void testPopAsync() throws RemotingException, InterruptedException, MQClientException { + PopCallback popCallback = mock(PopCallback.class); + when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPIImpl); + pullAPIWrapper.popAsync(createMessageQueue(), + System.currentTimeMillis(), + 1, + defaultGroup, + defaultTimeout, + popCallback, + true, + 1, + false, + "", + ""); + verify(mqClientAPIImpl, times(1)).popMessageAsync(eq(defaultBroker), + eq(defaultBrokerAddr), + any(PopMessageRequestHeader.class), + eq(13000L), + any(PopCallback.class)); + } + + private ConcurrentMap createTopicRouteTable() { + TopicRouteData topicRouteData = new TopicRouteData(); + List brokerDatas = new ArrayList<>(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName(defaultBroker); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, defaultBroker); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDatas.add(brokerData); + topicRouteData.setBrokerDatas(brokerDatas); + HashMap> filterServerTable = new HashMap<>(); + List filterServers = new ArrayList<>(); + filterServers.add(defaultBroker); + filterServerTable.put(defaultBrokerAddr, filterServers); + topicRouteData.setFilterServerTable(filterServerTable); + ConcurrentMap result = new ConcurrentHashMap<>(); + result.put(defaultTopic, topicRouteData); + return result; + } + + private MessageQueue createMessageQueue() { + MessageQueue result = new MessageQueue(); + result.setQueueId(0); + result.setBrokerName(defaultBroker); + result.setTopic(defaultTopic); + return result; + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + long curTime = System.currentTimeMillis(); + result.setBornTimestamp(curTime - 1000); + result.getProperties().put(MessageConst.PROPERTY_POP_CK, curTime + " " + curTime + " " + curTime + " " + curTime); + result.setKeys("keys"); + result.setSysFlag(MessageSysFlag.INNER_BATCH_FLAG); + result.setSysFlag(result.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setBornHost(bornHost); + result.setStoreHost(storeHost); + return result; + } +} From 524ca4e862b3bb4b5f713ba09dac106beff0ce98 Mon Sep 17 00:00:00 2001 From: imzs Date: Wed, 14 Aug 2024 13:48:44 +0800 Subject: [PATCH 103/265] [ISSUE #8460] Improve the pop revive process when reading biz messages from a remote broker - part2 (#8494) --- .../broker/failover/EscapeBridge.java | 39 +++++-- .../broker/processor/PopReviveService.java | 22 ++-- .../broker/failover/EscapeBridgeTest.java | 106 ++++++++++++++++-- .../processor/PopReviveServiceTest.java | 45 +++++++- .../apache/rocketmq/common/BrokerConfig.java | 11 ++ 5 files changed, 196 insertions(+), 27 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java index 7df49f8c470..762d917d640 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java @@ -48,9 +48,11 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.tieredstore.TieredMessageStore; public class EscapeBridge { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -99,7 +101,7 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) { try { messageExt.setWaitStoreMsgOK(false); - final SendResult sendResult = putMessageToRemoteBroker(messageExt); + final SendResult sendResult = putMessageToRemoteBroker(messageExt, null); return transformSendResult2PutResult(sendResult); } catch (Exception e) { LOG.error("sendMessageInFailover to remote failed", e); @@ -112,7 +114,10 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) { } } - private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) { + public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, String brokerNameToSend) { + if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { // not remote broker + return null; + } final boolean isTransHalfMessage = TransactionalMessageUtil.buildHalfTopic().equals(messageExt.getTopic()); MessageExtBrokerInner messageToPut = messageExt; if (isTransHalfMessage) { @@ -125,12 +130,26 @@ private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) { return null; } - final MessageQueue mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName()); - - messageToPut.setQueueId(mqSelected.getQueueId()); + final MessageQueue mqSelected; + if (StringUtils.isEmpty(brokerNameToSend)) { + mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName()); + messageToPut.setQueueId(mqSelected.getQueueId()); + brokerNameToSend = mqSelected.getBrokerName(); + if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { + LOG.warn("putMessageToRemoteBroker failed, remote broker not found. Topic: {}, MsgId: {}, Broker: {}", + messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); + return null; + } + } else { + mqSelected = new MessageQueue(messageExt.getTopic(), brokerNameToSend, messageExt.getQueueId()); + } - final String brokerNameToSend = mqSelected.getBrokerName(); final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend); + if (null == brokerAddrToSend) { + LOG.warn("putMessageToRemoteBroker failed, remote broker address not found. Topic: {}, MsgId: {}, Broker: {}", + messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); + return null; + } final long beginTimestamp = System.currentTimeMillis(); try { @@ -279,8 +298,12 @@ public CompletableFuture> getMessageAsync(St } List list = decodeMsgList(result, deCompressBody); if (list == null || list.isEmpty()) { - LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, result); - return Triple.of(null, "Can not get msg", false); // local store, so no retry + // OFFSET_FOUND_NULL returned by TieredMessageStore indicates exception occurred + boolean needRetry = GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus()) + && messageStore instanceof TieredMessageStore; + LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, needRetry {}, result is {}", + topic, offset, queueId, needRetry, result); + return Triple.of(null, "Can not get msg", needRetry); } return Triple.of(list.get(0), "", false); }); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 114d094600e..4b141d29102 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -199,9 +199,8 @@ private boolean reachTail(PullResult pullResult, long offset) { } // Triple - private CompletableFuture> getBizMessage(String topic, long offset, int queueId, - String brokerName) { - return this.brokerController.getEscapeBridge().getMessageAsync(topic, offset, queueId, brokerName, false); + public CompletableFuture> getBizMessage(PopCheckPoint popCheckPoint, long offset) { + return this.brokerController.getEscapeBridge().getMessageAsync(popCheckPoint.getTopic(), offset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName(), false); } public PullResult getMessage(String group, String topic, int queueId, long offset, int nums, @@ -358,7 +357,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { if (point.getTopic() == null || point.getCId() == null) { continue; } - map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(), point); + map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName(), point); PopMetricsManager.incPopReviveCkGetCount(point, queueId); point.setReviveOffset(messageExt.getQueueOffset()); if (firstRt == 0) { @@ -371,7 +370,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { } AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class); PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId); - String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime(); + String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + ackMsg.getBrokerName(); PopCheckPoint point = map.get(mergeKey); if (point == null) { if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) { @@ -396,7 +395,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { BatchAckMsg bAckMsg = JSON.parseObject(raw, BatchAckMsg.class); PopMetricsManager.incPopReviveAckGetCount(bAckMsg, queueId); - String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime(); + String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + bAckMsg.getBrokerName(); PopCheckPoint point = map.get(mergeKey); if (point == null) { if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) { @@ -528,7 +527,7 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { // retry msg long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j); - CompletableFuture> future = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName()) + CompletableFuture> future = getBizMessage(popCheckPoint, msgOffset) .thenApply(rst -> { MessageExt message = rst.getLeft(); if (message == null) { @@ -568,9 +567,9 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { private void rePutCK(PopCheckPoint oldCK, Pair pair) { int rePutTimes = oldCK.parseRePutTimes(); - if (rePutTimes >= ckRewriteIntervalsInSeconds.length) { - POP_LOGGER.warn("rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}", oldCK.getTopic(), oldCK.getCId(), - oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime()); + if (rePutTimes >= ckRewriteIntervalsInSeconds.length && brokerController.getBrokerConfig().isSkipWhenCKRePutReachMaxTimes()) { + POP_LOGGER.warn("rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}, {}", oldCK.getTopic(), oldCK.getCId(), + oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime(), rePutTimes); return; } @@ -588,7 +587,8 @@ private void rePutCK(PopCheckPoint oldCK, Pair pair) { newCk.setRePutTimes(String.valueOf(rePutTimes + 1)); // always increment even if removed from reviveRequestMap if (oldCK.getReviveTime() <= System.currentTimeMillis()) { // never expect an ACK matched in the future, we just use it to rewrite CK and try to revive retry message next time - newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[rePutTimes] * 1000); + int intervalIndex = rePutTimes >= ckRewriteIntervalsInSeconds.length ? ckRewriteIntervalsInSeconds.length - 1 : rePutTimes; + newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[intervalIndex] * 1000); } MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); brokerController.getMessageStore().putMessage(ckMsg); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java index 7ea06665c3e..27fc37dbec8 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/failover/EscapeBridgeTest.java @@ -30,19 +30,23 @@ import org.apache.rocketmq.broker.topic.TopicRouteInfoManager; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.apache.rocketmq.store.logfile.MappedFile; +import org.apache.rocketmq.tieredstore.TieredMessageStore; import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assert; @@ -58,6 +62,9 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) public class EscapeBridgeTest { @@ -75,6 +82,9 @@ public class EscapeBridgeTest { @Mock private DefaultMessageStore defaultMessageStore; + @Mock + private TieredMessageStore tieredMessageStore; + private GetMessageResult getMessageResult; @Mock @@ -200,14 +210,37 @@ public void getMessageAsyncTest_localStore_getMessageAsync_null() { } @Test - public void getMessageAsyncTest_localStore_decodeNothing() throws Exception { + public void getMessageAsyncTest_localStore_decodeNothing_DefaultMessageStore() throws Exception { when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(defaultMessageStore); - when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) - .thenReturn(CompletableFuture.completedFuture(mockGetMessageResult(0, TEST_TOPIC, null))); - Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); - Assert.assertNull(rst.getLeft()); - Assert.assertEquals("Can not get msg", rst.getMiddle()); - Assert.assertFalse(rst.getRight()); // no retry + for (GetMessageStatus status : GetMessageStatus.values()) { + GetMessageResult getMessageResult = mockGetMessageResult(0, TEST_TOPIC, null); + getMessageResult.setStatus(status); + when(defaultMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(getMessageResult)); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("Can not get msg", rst.getMiddle()); + Assert.assertFalse(rst.getRight()); // DefaultMessageStore, no retry + } + } + + @Test + public void getMessageAsyncTest_localStore_decodeNothing_TieredMessageStore() throws Exception { + when(brokerController.getMessageStoreByBrokerName(any())).thenReturn(tieredMessageStore); + for (GetMessageStatus status : GetMessageStatus.values()) { + GetMessageResult getMessageResult = new GetMessageResult(); + getMessageResult.setStatus(status); + when(tieredMessageStore.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(getMessageResult)); + Triple rst = escapeBridge.getMessageAsync(TEST_TOPIC, 0, DEFAULT_QUEUE_ID, BROKER_NAME, false).join(); + Assert.assertNull(rst.getLeft()); + Assert.assertEquals("Can not get msg", rst.getMiddle()); + if (GetMessageStatus.OFFSET_FOUND_NULL.equals(status)) { + Assert.assertTrue(rst.getRight()); // TieredMessageStore returns OFFSET_FOUND_NULL, need retry + } else { + Assert.assertFalse(rst.getRight()); // other status, like DefaultMessageStore, no retry + } + } } @Test @@ -320,6 +353,57 @@ public void decodeMsgListTest_messageNotNull() throws Exception { Assert.assertTrue(Arrays.equals(msg.getBody(), list.get(0).getBody())); } + @Test + public void testPutMessageToRemoteBroker_noSpecificBrokerName_hasRemoteBroker() throws Exception { + MessageExtBrokerInner message = new MessageExtBrokerInner(); + message.setTopic(TEST_TOPIC); + String anotherBrokerName = "broker_b"; + TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName); + when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo); + when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn("127.0.0.1"); + escapeBridge.putMessageToRemoteBroker(message, null); + verify(brokerOuterAPI).sendMessageToSpecificBroker(eq("127.0.0.1"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong()); + } + + @Test + public void testPutMessageToRemoteBroker_noSpecificBrokerName_noRemoteBroker() throws Exception { + MessageExtBrokerInner message = new MessageExtBrokerInner(); + message.setTopic(TEST_TOPIC); + TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME); + when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo); + escapeBridge.putMessageToRemoteBroker(message, null); + verify(topicRouteInfoManager, times(0)).findBrokerAddressInPublish(anyString()); + } + + @Test + public void testPutMessageToRemoteBroker_specificBrokerName_equals() throws Exception { + escapeBridge.putMessageToRemoteBroker(new MessageExtBrokerInner(), BROKER_NAME); + verify(topicRouteInfoManager, times(0)).tryToFindTopicPublishInfo(anyString()); + } + + @Test + public void testPutMessageToRemoteBroker_specificBrokerName_addressNotFound() throws Exception { + MessageExtBrokerInner message = new MessageExtBrokerInner(); + message.setTopic(TEST_TOPIC); + TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME); + when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo); + escapeBridge.putMessageToRemoteBroker(message, "whatever"); + verify(topicRouteInfoManager).findBrokerAddressInPublish(eq("whatever")); + verify(brokerOuterAPI, times(0)).sendMessageToSpecificBroker(anyString(), anyString(), any(MessageExtBrokerInner.class), anyString(), anyLong()); + } + + @Test + public void testPutMessageToRemoteBroker_specificBrokerName_addressFound() throws Exception { + MessageExtBrokerInner message = new MessageExtBrokerInner(); + message.setTopic(TEST_TOPIC); + String anotherBrokerName = "broker_b"; + TopicPublishInfo publishInfo = mockTopicPublishInfo(BROKER_NAME, anotherBrokerName); + when(topicRouteInfoManager.tryToFindTopicPublishInfo(anyString())).thenReturn(publishInfo); + when(topicRouteInfoManager.findBrokerAddressInPublish(anotherBrokerName)).thenReturn("127.0.0.1"); + escapeBridge.putMessageToRemoteBroker(message, anotherBrokerName); + verify(brokerOuterAPI).sendMessageToSpecificBroker(eq("127.0.0.1"), eq(anotherBrokerName), any(MessageExtBrokerInner.class), anyString(), anyLong()); + } + private GetMessageResult mockGetMessageResult(int count, String topic, byte[] body) throws Exception { GetMessageResult result = new GetMessageResult(); for (int i = 0; i < count; i++) { @@ -337,4 +421,12 @@ private GetMessageResult mockGetMessageResult(int count, String topic, byte[] bo return result; } + private TopicPublishInfo mockTopicPublishInfo(String... brokerNames) { + TopicPublishInfo topicPublishInfo = new TopicPublishInfo(); + for (String brokerName : brokerNames) { + topicPublishInfo.getMessageQueueList().add(new MessageQueue(TEST_TOPIC, brokerName, 0)); + } + return topicPublishInfo; + } + } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java index d7ea97c5502..3010e836101 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java @@ -104,7 +104,6 @@ public void before() { brokerConfig = new BrokerConfig(); brokerConfig.setBrokerClusterName(CLUSTER_NAME); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); - when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); when(brokerController.getMessageStore()).thenReturn(messageStore); when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); @@ -285,6 +284,7 @@ public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK() throws @Test public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_end() throws Throwable { + brokerConfig.setSkipWhenCKRePutReachMaxTimes(true); PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); ck.setRePutTimes("17"); PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); @@ -306,6 +306,30 @@ public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_end() th verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK } + @Test + public void testReviveMsgFromCk_messageFound_writeRetryFailed_rewriteCK_noEnd() throws Throwable { + brokerConfig.setSkipWhenCKRePutReachMaxTimes(false); + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + ck.setRePutTimes(Byte.MAX_VALUE + ""); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + StringBuilder actualRetryTopic = new StringBuilder(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(new MessageExt(), "", false))); + when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenAnswer(invocation -> { + MessageExtBrokerInner msg = invocation.getArgument(0); + actualRetryTopic.append(msg.getTopic()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED)); + }); + + popReviveService.mergeAndRevive(reviveObj); + Assert.assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, GROUP, false), actualRetryTopic.toString()); + verify(escapeBridge, times(1)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + @Test public void testReviveMsgFromCk_messageNotFound_noRetry() throws Throwable { PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); @@ -349,6 +373,7 @@ public void testReviveMsgFromCk_messageNotFound_needRetry() throws Throwable { @Test public void testReviveMsgFromCk_messageNotFound_needRetry_end() throws Throwable { + brokerConfig.setSkipWhenCKRePutReachMaxTimes(true); PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); ck.setRePutTimes("17"); PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); @@ -363,6 +388,23 @@ public void testReviveMsgFromCk_messageNotFound_needRetry_end() throws Throwable verify(messageStore, times(0)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK } + @Test + public void testReviveMsgFromCk_messageNotFound_needRetry_noEnd() throws Throwable { + brokerConfig.setSkipWhenCKRePutReachMaxTimes(false); + PopCheckPoint ck = buildPopCheckPoint(0, 0, 0); + ck.setRePutTimes(Byte.MAX_VALUE + ""); + PopReviveService.ConsumeReviveObj reviveObj = new PopReviveService.ConsumeReviveObj(); + reviveObj.map.put("", ck); + reviveObj.endTime = System.currentTimeMillis(); + + when(escapeBridge.getMessageAsync(anyString(), anyLong(), anyInt(), anyString(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(Triple.of(null, "", true))); + + popReviveService.mergeAndRevive(reviveObj); + verify(escapeBridge, times(0)).putMessageToSpecificQueue(any(MessageExtBrokerInner.class)); // write retry + verify(messageStore, times(1)).putMessage(any(MessageExtBrokerInner.class)); // rewrite CK + } + public static PopCheckPoint buildPopCheckPoint(long startOffset, long popTime, long reviveOffset) { PopCheckPoint ck = new PopCheckPoint(); ck.setStartOffset(startOffset); @@ -386,6 +428,7 @@ public static AckMsg buildAckMsg(long offset, long popTime) { ackMsg.setTopic(TOPIC); ackMsg.setQueueId(0); ackMsg.setPopTime(popTime); + ackMsg.setBrokerName("broker-a"); return ackMsg; } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 8982e59d03e..10bf7f76e86 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -419,6 +419,9 @@ public class BrokerConfig extends BrokerIdentity { */ private String configBlackList = "configBlackList;brokerConfigPath"; + // if false, will still rewrite ck after max times 17 + private boolean skipWhenCKRePutReachMaxTimes = false; + public String getConfigBlackList() { return configBlackList; } @@ -1826,4 +1829,12 @@ public boolean isEnablePopMessageThreshold() { public void setEnablePopMessageThreshold(boolean enablePopMessageThreshold) { this.enablePopMessageThreshold = enablePopMessageThreshold; } + + public boolean isSkipWhenCKRePutReachMaxTimes() { + return skipWhenCKRePutReachMaxTimes; + } + + public void setSkipWhenCKRePutReachMaxTimes(boolean skipWhenCKRePutReachMaxTimes) { + this.skipWhenCKRePutReachMaxTimes = skipWhenCKRePutReachMaxTimes; + } } From 227be2cf91f1adb767632159eaffcfb3238adc56 Mon Sep 17 00:00:00 2001 From: bxfjb <48467309+bxfjb@users.noreply.github.com> Date: Wed, 14 Aug 2024 23:42:26 +0800 Subject: [PATCH 104/265] [ISSUE #8532] Fix flush metadata when commit file because of full file (#8533) --- .../org/apache/rocketmq/tieredstore/file/FlatAppendFile.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java index 0c20a1cfb4f..891170d703a 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatAppendFile.java @@ -180,6 +180,7 @@ public AppendResult append(ByteBuffer buffer, long timestamp) { log.info("FlatAppendFile#append not successful for the file {} is full, commit result={}", fileSegment.getPath(), commitResult); if (commitResult) { + this.flushFileSegmentMeta(fileSegment); return this.rollingNewFile(this.getAppendOffset()).append(buffer, timestamp); } else { return AppendResult.UNKNOWN_ERROR; From 7964f06f273351cdad28f1332ce28e52693ee7a6 Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Thu, 15 Aug 2024 10:39:27 +0800 Subject: [PATCH 105/265] [ISSUE #8531]Update jaeger-thrift, exclude unnecessary tomcat-embed-core --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 03a4ad18a13..41fc3db0c40 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ 1.5.2-2 1.8.0 0.33.0 - 1.6.0 + 1.8.1 0.3.1.2 6.0.53 1.0-beta-4 From aa788e87966d85176d682fc6a84d144a1aa20707 Mon Sep 17 00:00:00 2001 From: zekai-li <58294989+zekai-li@users.noreply.github.com> Date: Fri, 16 Aug 2024 00:33:35 +0800 Subject: [PATCH 106/265] [ISSUE #8289] Fixed network bugs and merged networkutil code (#8290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ISSUE #8289] Fixed network bugs and merged networkutil code Fix the following three as 1，TraceBean： private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP()); getIP mayby ipv6 2，NetworkUtil：if (ip.startsWith("127.0") || ip.startsWith("192.168") || ip.startsWith("0."))；Check whether Intranet errors exist * [ISSUE #8289] Fixed network bugs and merged networkutil code Fix the following three as 1，TraceBean： private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP()); getIP mayby ipv6 2，NetworkUtil：if (ip.startsWith("127.0") || ip.startsWith("192.168") || ip.startsWith("0."))；Check whether Intranet errors exist --- .../rocketmq/client/trace/TraceBean.java | 11 +- .../org/apache/rocketmq/common/UtilAll.java | 35 +++--- .../rocketmq/common/utils/NetworkUtil.java | 109 +++++++++++------- 3 files changed, 92 insertions(+), 63 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java index 70c147e1eb3..17db1fbfa15 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceBean.java @@ -21,7 +21,7 @@ import org.apache.rocketmq.common.message.MessageType; public class TraceBean { - private static final String LOCAL_ADDRESS = UtilAll.ipToIPv4Str(UtilAll.getIP()); + private static final String LOCAL_ADDRESS; private String topic = ""; private String msgId = ""; private String offsetMsgId = ""; @@ -37,6 +37,15 @@ public class TraceBean { private String transactionId; private boolean fromTransactionCheck; + static { + byte[] ip = UtilAll.getIP(); + if (ip.length == 4) { + LOCAL_ADDRESS = UtilAll.ipToIPv4Str(ip); + } else { + LOCAL_ADDRESS = UtilAll.ipToIPv6Str(ip); + } + } + public MessageType getMsgType() { return msgType; } diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index 3629ae648bb..a42ac3f364d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -326,16 +326,14 @@ public static String bytes2string(byte[] src) { } public static void writeInt(char[] buffer, int pos, int value) { - char[] hexArray = HEX_ARRAY; for (int moveBits = 28; moveBits >= 0; moveBits -= 4) { - buffer[pos++] = hexArray[(value >>> moveBits) & 0x0F]; + buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F]; } } public static void writeShort(char[] buffer, int pos, int value) { - char[] hexArray = HEX_ARRAY; for (int moveBits = 12; moveBits >= 0; moveBits -= 4) { - buffer[pos++] = hexArray[(value >>> moveBits) & 0x0F]; + buffer[pos++] = HEX_ARRAY[(value >>> moveBits) & 0x0F]; } } @@ -536,25 +534,18 @@ public static boolean isInternalIP(byte[] ip) { } else if (ip[0] == (byte) 127) { return true; } else if (ip[0] == (byte) 172) { - if (ip[1] >= (byte) 16 && ip[1] <= (byte) 31) { - return true; - } + return ip[1] >= (byte) 16 && ip[1] <= (byte) 31; } else if (ip[0] == (byte) 192) { - if (ip[1] == (byte) 168) { - return true; - } + return ip[1] == (byte) 168; } return false; } public static boolean isInternalV6IP(InetAddress inetAddr) { - if (inetAddr.isAnyLocalAddress() // Wild card ipv6 + return inetAddr.isAnyLocalAddress() // Wild card ipv6 || inetAddr.isLinkLocalAddress() // Single broadcast ipv6 address: fe80:xx:xx... || inetAddr.isLoopbackAddress() //Loopback ipv6 address - || inetAddr.isSiteLocalAddress()) { // Site local ipv6 address: fec0:xx:xx... - return true; - } - return false; + || inetAddr.isSiteLocalAddress();// Site local ipv6 address: fec0:xx:xx... } private static boolean ipCheck(byte[] ip) { @@ -605,15 +596,15 @@ public static String ipToIPv6Str(byte[] ip) { public static byte[] getIP() { try { - Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces(); - InetAddress ip = null; + Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces(); + InetAddress ip; byte[] internalIP = null; while (allNetInterfaces.hasMoreElements()) { - NetworkInterface netInterface = (NetworkInterface) allNetInterfaces.nextElement(); - Enumeration addresses = netInterface.getInetAddresses(); + NetworkInterface netInterface = allNetInterfaces.nextElement(); + Enumeration addresses = netInterface.getInetAddresses(); while (addresses.hasMoreElements()) { - ip = (InetAddress) addresses.nextElement(); - if (ip != null && ip instanceof Inet4Address) { + ip = addresses.nextElement(); + if (ip instanceof Inet4Address) { byte[] ipByte = ip.getAddress(); if (ipByte.length == 4) { if (ipCheck(ipByte)) { @@ -624,7 +615,7 @@ public static byte[] getIP() { } } } - } else if (ip != null && ip instanceof Inet6Address) { + } else if (ip instanceof Inet6Address) { byte[] ipByte = ip.getAddress(); if (ipByte.length == 16) { if (ipV6Check(ipByte)) { diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java index 7dd83e61799..a7a9a7c7960 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java @@ -19,15 +19,21 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Method; +import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.NetworkInterface; import java.net.SocketAddress; +import java.net.SocketException; import java.nio.channels.Selector; import java.nio.channels.spi.SelectorProvider; import java.util.ArrayList; import java.util.Enumeration; +import java.util.List; + +import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -59,18 +65,14 @@ public static Selector openSelector() throws IOException { if (isLinuxPlatform()) { try { final Class providerClazz = Class.forName("sun.nio.ch.EPollSelectorProvider"); - if (providerClazz != null) { - try { - final Method method = providerClazz.getMethod("provider"); - if (method != null) { - final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null); - if (selectorProvider != null) { - result = selectorProvider.openSelector(); - } - } - } catch (final Exception e) { - log.warn("Open ePoll Selector for linux platform exception", e); + try { + final Method method = providerClazz.getMethod("provider"); + final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null); + if (selectorProvider != null) { + result = selectorProvider.openSelector(); } + } catch (final Exception e) { + log.warn("Open ePoll Selector for linux platform exception", e); } } catch (final Exception e) { // ignore @@ -88,48 +90,71 @@ public static boolean isLinuxPlatform() { return isLinuxPlatform; } - public static String getLocalAddress() { - try { - // Traversal Network interface to get the first non-loopback and non-private address - Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); - ArrayList ipv4Result = new ArrayList<>(); - ArrayList ipv6Result = new ArrayList<>(); - while (enumeration.hasMoreElements()) { - final NetworkInterface nif = enumeration.nextElement(); - if (isBridge(nif) || nif.isVirtual() || nif.isPointToPoint() || !nif.isUp()) { - continue; - } - - final Enumeration en = nif.getInetAddresses(); - while (en.hasMoreElements()) { - final InetAddress address = en.nextElement(); - if (!address.isLoopbackAddress()) { - if (address instanceof Inet6Address) { - ipv6Result.add(normalizeHostAddress(address)); - } else { - ipv4Result.add(normalizeHostAddress(address)); + public static List getLocalInetAddressList() throws SocketException { + Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); + List inetAddressList = new ArrayList<>(); + // Traversal Network interface to get the non-bridge and non-virtual and non-ppp and up address + while (enumeration.hasMoreElements()) { + final NetworkInterface nif = enumeration.nextElement(); + if (isBridge(nif) || nif.isVirtual() || nif.isPointToPoint() || !nif.isUp()) { + continue; + } + InetAddressValidator validator = InetAddressValidator.getInstance(); + final Enumeration en = nif.getInetAddresses(); + while (en.hasMoreElements()) { + final InetAddress address = en.nextElement(); + if (address instanceof Inet4Address) { + byte[] ipByte = address.getAddress(); + if (ipByte.length == 4) { + if (validator.isValidInet4Address(UtilAll.ipToIPv4Str(ipByte))) { + inetAddressList.add(address); + } + } + } else if (address instanceof Inet6Address) { + byte[] ipByte = address.getAddress(); + if (ipByte.length == 16) { + if (validator.isValidInet6Address(UtilAll.ipToIPv6Str(ipByte))) { + inetAddressList.add(address); } } } } + } + return inetAddressList; + } - // prefer ipv4 + public static InetAddress getLocalInetAddress() { + try { + ArrayList ipv4Result = new ArrayList<>(); + ArrayList ipv6Result = new ArrayList<>(); + List localInetAddressList = getLocalInetAddressList(); + for (InetAddress inetAddress : localInetAddressList) { + if (inetAddress instanceof Inet6Address) { + ipv6Result.add(inetAddress); + } else { + ipv4Result.add(inetAddress); + } + } + // prefer ipv4 and prefer external ip if (!ipv4Result.isEmpty()) { - for (String ip : ipv4Result) { - if (ip.startsWith("127.0") || ip.startsWith("192.168") || ip.startsWith("0.")) { + for (InetAddress ip : ipv4Result) { + if (UtilAll.isInternalIP(ip.getAddress())) { continue; } - return ip; } - return ipv4Result.get(ipv4Result.size() - 1); } else if (!ipv6Result.isEmpty()) { + for (InetAddress ip : ipv6Result) { + if (UtilAll.isInternalV6IP(ip)) { + continue; + } + return ip; + } return ipv6Result.get(0); } //If failed to find,fall back to localhost - final InetAddress localHost = InetAddress.getLocalHost(); - return normalizeHostAddress(localHost); + return InetAddress.getLocalHost(); } catch (Exception e) { log.error("Failed to obtain local address", e); } @@ -137,6 +162,11 @@ public static String getLocalAddress() { return null; } + public static String getLocalAddress() { + InetAddress localHost = getLocalInetAddress(); + return normalizeHostAddress(localHost); + } + public static String normalizeHostAddress(final InetAddress localHost) { if (localHost instanceof Inet6Address) { return "[" + localHost.getHostAddress() + "]"; @@ -149,8 +179,7 @@ public static SocketAddress string2SocketAddress(final String addr) { int split = addr.lastIndexOf(":"); String host = addr.substring(0, split); String port = addr.substring(split + 1); - InetSocketAddress isa = new InetSocketAddress(host, Integer.parseInt(port)); - return isa; + return new InetSocketAddress(host, Integer.parseInt(port)); } public static String socketAddress2String(final SocketAddress addr) { From 9059fcddd8f56792957c6c8ddbe71bc0815fa8eb Mon Sep 17 00:00:00 2001 From: Hard_X <111902269+HardX8@users.noreply.github.com> Date: Fri, 16 Aug 2024 14:30:32 +0800 Subject: [PATCH 107/265] [ISSUE #8519] Add test case for rocketmq acl module. (#8508) * add AuthorizationHeaderTest * add PlainPermissionCheckerTest * add license header for AuthorizationHeaderTest and PlainPermissionCheckerTest * Update --- .../acl/common/AuthorizationHeaderTest.java | 64 ++++++++++++++ .../acl/plain/PlainPermissionCheckerTest.java | 88 +++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 acl/src/test/java/org/apache/rocketmq/acl/common/AuthorizationHeaderTest.java create mode 100644 acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionCheckerTest.java diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/AuthorizationHeaderTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/AuthorizationHeaderTest.java new file mode 100644 index 00000000000..bb735f0a045 --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/AuthorizationHeaderTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.common; + +import org.junit.Before; +import org.junit.Test; +import org.junit.Assert; + +public class AuthorizationHeaderTest { + + private static final String AUTH_HEADER = "Signature Credential=1234567890/test, SignedHeaders=host, Signature=1234567890"; + private AuthorizationHeader authorizationHeader; + + @Before + public void setUp() throws Exception { + authorizationHeader = new AuthorizationHeader(AUTH_HEADER); + } + + @Test + public void testGetMethod() { + Assert.assertEquals("Signature", authorizationHeader.getMethod()); + } + + @Test + public void testGetAccessKey() { + Assert.assertEquals("1234567890", authorizationHeader.getAccessKey()); + } + + @Test + public void testGetSignedHeaders() { + String[] expectedHeaders = {"host"}; + Assert.assertArrayEquals(expectedHeaders, authorizationHeader.getSignedHeaders()); + } + + @Test + public void testGetSignature() { + Assert.assertEquals("EjRWeJA=", authorizationHeader.getSignature()); + } + + @Test(expected = Exception.class) + public void testInvalidAuthorizationHeader() throws Exception { + new AuthorizationHeader("Invalid Header"); + } + + @Test(expected = Exception.class) + public void testMalformedAuthorizationHeader() throws Exception { + new AuthorizationHeader("Malformed, Header"); + } + +} diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionCheckerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionCheckerTest.java new file mode 100644 index 00000000000..4df0ea5d2d6 --- /dev/null +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionCheckerTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.acl.plain; + +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.Permission; +import org.junit.Before; +import org.junit.Test; +import org.junit.Assert; + +public class PlainPermissionCheckerTest { + + private PlainPermissionChecker permissionChecker; + + @Before + public void setUp() { + permissionChecker = new PlainPermissionChecker(); + } + + @Test + public void testCheck_withAdminPermission_shouldPass() { + PlainAccessResource checkedAccess = new PlainAccessResource(); + checkedAccess.setRequestCode(Permission.SUB); + checkedAccess.addResourceAndPerm("topic1", Permission.PUB); + PlainAccessResource ownedAccess = new PlainAccessResource(); + ownedAccess.setAccessKey("adminUser"); + ownedAccess.setAdmin(true); + try { + permissionChecker.check(checkedAccess, ownedAccess); + } catch (AclException e) { + Assert.fail("Should not throw any exception for admin user"); + } + } + + @Test(expected = AclException.class) + public void testCheck_withoutAdminPermissionAndNoDefaultPerm_shouldThrowAclException() { + PlainAccessResource checkedAccess = new PlainAccessResource(); + checkedAccess.setRequestCode(Permission.SUB); + checkedAccess.addResourceAndPerm("topic1", Permission.PUB); + PlainAccessResource ownedAccess = new PlainAccessResource(); + ownedAccess.setAccessKey("nonAdminUser"); + ownedAccess.setAdmin(false); + permissionChecker.check(checkedAccess, ownedAccess); + } + + @Test + public void testCheck_withDefaultPermissions_shouldPass() { + PlainAccessResource checkedAccess = new PlainAccessResource(); + checkedAccess.setRequestCode(Permission.SUB); + checkedAccess.addResourceAndPerm("topic1", Permission.PUB); + PlainAccessResource ownedAccess = new PlainAccessResource(); + ownedAccess.setAccessKey("nonAdminUser"); + ownedAccess.setAdmin(false); + ownedAccess.setDefaultTopicPerm(Permission.PUB); + try { + permissionChecker.check(checkedAccess, ownedAccess); + } catch (AclException e) { + Assert.fail("Should not throw any exception for default permissions"); + } + } + + @Test(expected = AclException.class) + public void testCheck_withoutPermission_shouldThrowAclException() { + PlainAccessResource checkedAccess = new PlainAccessResource(); + checkedAccess.setRequestCode(Permission.SUB); + checkedAccess.addResourceAndPerm("topic1", Permission.PUB); + PlainAccessResource ownedAccess = new PlainAccessResource(); + ownedAccess.setAccessKey("nonAdminUser"); + ownedAccess.setAdmin(false); + ownedAccess.setDefaultTopicPerm(Permission.SUB); + permissionChecker.check(checkedAccess, ownedAccess); + } + +} From 2f482e2ff95cb3c1917d6758aa7b5b9811137ae4 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 16 Aug 2024 14:31:03 +0800 Subject: [PATCH 108/265] [ISSUE #8517] Add more test coverage for PullMessageService (#8542) --- .../impl/consumer/PullMessageServiceTest.java | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java new file mode 100644 index 00000000000..73fa4e95d69 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/PullMessageServiceTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.message.MessageRequestMode; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PullMessageServiceTest { + + @Mock + private MQClientInstance mQClientFactory; + + @Mock + private ScheduledExecutorService executorService; + + private PullMessageService pullMessageService; + + private final long defaultTimeout = 3000L; + + private final String defaultGroup = "defaultGroup"; + + @Before + public void init() throws Exception { + pullMessageService = new PullMessageService(mQClientFactory); + FieldUtils.writeDeclaredField(pullMessageService, "scheduledExecutorService", executorService, true); + pullMessageService.start(); + } + + @Test + public void testProcessPullResult() { + PopRequest popRequest = mock(PopRequest.class); + pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout); + pullMessageService.makeStop(); + pullMessageService.executePopPullRequestLater(popRequest, defaultTimeout); + verify(executorService, times(1)) + .schedule(any(Runnable.class), + eq(defaultTimeout), + eq(TimeUnit.MILLISECONDS)); + } + + @Test + public void testExecutePopPullRequestImmediately() throws IllegalAccessException, InterruptedException { + PopRequest popRequest = mock(PopRequest.class); + LinkedBlockingQueue messageRequestQueue = mock(LinkedBlockingQueue.class); + FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true); + pullMessageService.executePopPullRequestImmediately(popRequest); + verify(messageRequestQueue, times(1)).put(any(PopRequest.class)); + } + + @Test + public void testExecuteTaskLater() { + Runnable runnable = mock(Runnable.class); + pullMessageService.executeTaskLater(runnable, defaultTimeout); + pullMessageService.makeStop(); + pullMessageService.executeTaskLater(runnable, defaultTimeout); + verify(executorService, times(1)) + .schedule(any(Runnable.class), + eq(defaultTimeout), + eq(TimeUnit.MILLISECONDS)); + } + + @Test + public void testExecuteTask() { + Runnable runnable = mock(Runnable.class); + pullMessageService.executeTask(runnable); + pullMessageService.makeStop(); + pullMessageService.executeTask(runnable); + verify(executorService, times(1)).execute(any(Runnable.class)); + } + + @Test + public void testGetScheduledExecutorService() { + assertEquals(executorService, pullMessageService.getScheduledExecutorService()); + } + + @Test + public void testRun() throws InterruptedException, IllegalAccessException { + LinkedBlockingQueue messageRequestQueue = new LinkedBlockingQueue<>(); + PopRequest popRequest = mock(PopRequest.class); + when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + messageRequestQueue.put(popRequest); + DefaultMQPushConsumerImpl defaultMQPushConsumerImpl = mock(DefaultMQPushConsumerImpl.class); + when(mQClientFactory.selectConsumer(any())).thenReturn(defaultMQPushConsumerImpl); + FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true); + new Thread(() -> pullMessageService.run()).start(); + TimeUnit.SECONDS.sleep(1); + pullMessageService.makeStop(); + verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup)); + verify(defaultMQPushConsumerImpl).popMessage(any(PopRequest.class)); + } + + @Test + public void testRunWithNullConsumer() throws InterruptedException, IllegalAccessException { + LinkedBlockingQueue messageRequestQueue = new LinkedBlockingQueue<>(); + PopRequest popRequest = mock(PopRequest.class); + when(popRequest.getMessageRequestMode()).thenReturn(MessageRequestMode.POP); + when(popRequest.getConsumerGroup()).thenReturn(defaultGroup); + messageRequestQueue.put(popRequest); + FieldUtils.writeDeclaredField(pullMessageService, "messageRequestQueue", messageRequestQueue, true); + new Thread(() -> pullMessageService.run()).start(); + TimeUnit.SECONDS.sleep(1); + pullMessageService.makeStop(); + verify(mQClientFactory, times(1)).selectConsumer(eq(defaultGroup)); + } +} From b512ba1a28a2f406ea2354fbfa7deedcc05d9ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Fri, 16 Aug 2024 15:30:20 +0800 Subject: [PATCH 109/265] [ISSUE #8544] Add a retry mechanism to the unit test pipeline (#8545) * Add a retry mechanism to the unit test pipeline * Remove the permissions field --- .github/workflows/bazel.yml | 12 +++++++++++- .github/workflows/maven.yaml | 11 ++++++++++- .github/workflows/rerun-workflow.yml | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/rerun-workflow.yml diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index af674592bb9..73a49aa6a64 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -7,6 +7,7 @@ on: - master - develop - bazel + jobs: build: name: "bazel-compile (${{ matrix.os }})" @@ -19,4 +20,13 @@ jobs: - name: Build run: bazel build --config=remote //... - name: Run Tests - run: bazel test --config=remote //... \ No newline at end of file + run: bazel test --config=remote //... + - name: Retry if failed + # if it failed , retry 2 times at most + if: failure() && fromJSON(github.run_attempt) < 3 + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Attempting to retry workflow..." + gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 06db86e0157..3291d993ea0 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -31,4 +31,13 @@ jobs: with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log - retention-days: 1 \ No newline at end of file + retention-days: 1 + - name: Retry if failed + # if it failed , retry 2 times at most + if: failure() && fromJSON(github.run_attempt) < 3 + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Attempting to retry workflow..." + gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml new file mode 100644 index 00000000000..2f3258c1f6e --- /dev/null +++ b/.github/workflows/rerun-workflow.yml @@ -0,0 +1,18 @@ +name: Rerun workflow +on: + workflow_dispatch: + inputs: + run_id: + required: true + +jobs: + rerun: + runs-on: ubuntu-latest + steps: + - name: rerun ${{ inputs.run_id }} + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 + gh run rerun ${{ inputs.run_id }} --failed \ No newline at end of file From 39f289af243faa3aa8960cdfd82db7d316a30bc6 Mon Sep 17 00:00:00 2001 From: syhleo <757187389@qq.com> Date: Sun, 18 Aug 2024 15:19:21 +0800 Subject: [PATCH 110/265] [ISSUE #8547] Add UnitTest of ControllableOffset (#8548) --- .../store/ControllableOffsetTest.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java new file mode 100644 index 00000000000..23a56ca51ca --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/store/ControllableOffsetTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer.store; + + +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +public class ControllableOffsetTest { + + private ControllableOffset controllableOffset; + + @Before + public void setUp() { + controllableOffset = new ControllableOffset(0); + } + + @Test + public void testUpdateAndFreeze_ShouldFreezeOffsetAtTargetValue() { + controllableOffset.updateAndFreeze(100); + assertEquals(100, controllableOffset.getOffset()); + controllableOffset.update(200); + assertEquals(100, controllableOffset.getOffset()); + } + + @Test + public void testUpdate_ShouldUpdateOffsetWhenNotFrozen() { + controllableOffset.update(200); + assertEquals(200, controllableOffset.getOffset()); + } + + @Test + public void testUpdate_ShouldNotUpdateOffsetWhenFrozen() { + controllableOffset.updateAndFreeze(100); + controllableOffset.update(200); + assertEquals(100, controllableOffset.getOffset()); + } + + @Test + public void testUpdate_ShouldNotDecreaseOffsetWhenIncreaseOnly() { + controllableOffset.update(200); + controllableOffset.update(100, true); + assertEquals(200, controllableOffset.getOffset()); + } + + @Test + public void testUpdate_ShouldUpdateOffsetToGreaterValueWhenIncreaseOnly() { + controllableOffset.update(100); + controllableOffset.update(200, true); + assertEquals(200, controllableOffset.getOffset()); + } + +} From 2d901f16506966f23e189a15d7317601854f8489 Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 19 Aug 2024 14:55:35 +0800 Subject: [PATCH 111/265] [ISSUE #8551] Add more test coverage for AuthMigrator (#8552) --- .../auth/migration/AuthMigratorTest.java | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java diff --git a/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java b/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java new file mode 100644 index 00000000000..7a2bd5b2c76 --- /dev/null +++ b/auth/src/test/java/org/apache/rocketmq/auth/migration/AuthMigratorTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.auth.migration; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.acl.plain.PlainPermissionManager; +import org.apache.rocketmq.auth.authentication.manager.AuthenticationMetadataManager; +import org.apache.rocketmq.auth.authentication.model.User; +import org.apache.rocketmq.auth.authorization.manager.AuthorizationMetadataManager; +import org.apache.rocketmq.auth.config.AuthConfig; +import org.apache.rocketmq.common.AclConfig; +import org.apache.rocketmq.common.PlainAccessConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AuthMigratorTest { + + @Mock + private AuthenticationMetadataManager authenticationMetadataManager; + + @Mock + private AuthorizationMetadataManager authorizationMetadataManager; + + @Mock + private PlainPermissionManager plainPermissionManager; + + @Mock + private AuthConfig authConfig; + + private AuthMigrator authMigrator; + + @Before + public void setUp() throws IllegalAccessException { + when(authConfig.isMigrateAuthFromV1Enabled()).thenReturn(true); + authMigrator = new AuthMigrator(authConfig); + FieldUtils.writeDeclaredField(authMigrator, "authenticationMetadataManager", authenticationMetadataManager, true); + FieldUtils.writeDeclaredField(authMigrator, "authorizationMetadataManager", authorizationMetadataManager, true); + FieldUtils.writeDeclaredField(authMigrator, "plainPermissionManager", plainPermissionManager, true); + } + + @Test + public void testMigrateNoAclConfigDoesNothing() { + AclConfig aclConfig = mock(AclConfig.class); + when(aclConfig.getPlainAccessConfigs()).thenReturn(new ArrayList<>()); + when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig); + authMigrator.migrate(); + verify(authConfig, times(1)).isMigrateAuthFromV1Enabled(); + verify(plainPermissionManager, times(1)).getAllAclConfig(); + verify(authenticationMetadataManager, never()).createUser(any()); + verify(authorizationMetadataManager, never()).createAcl(any()); + } + + @Test + public void testMigrateWithAclConfigCreatesUserAndAcl() { + AclConfig aclConfig = mock(AclConfig.class); + List accessConfigs = new ArrayList<>(); + accessConfigs.add(createPlainAccessConfig()); + when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs); + when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig); + when(authenticationMetadataManager.getUser(anyString())) + .thenReturn(CompletableFuture.completedFuture(null)); + when(authenticationMetadataManager.createUser(any())) + .thenReturn(CompletableFuture.completedFuture(null)); + authMigrator.migrate(); + verify(authConfig, times(1)).isMigrateAuthFromV1Enabled(); + verify(plainPermissionManager, times(1)).getAllAclConfig(); + verify(authenticationMetadataManager, times(1)).createUser(any()); + verify(authorizationMetadataManager, times(1)).createAcl(any()); + } + + @Test + public void testMigrateExceptionInMigrateLogsError() { + PlainAccessConfig accessConfig = mock(PlainAccessConfig.class); + when(accessConfig.getAccessKey()).thenReturn("testAk"); + when(authenticationMetadataManager.createUser(any(User.class))) + .thenThrow(new RuntimeException("Test Exception")); + AclConfig aclConfig = mock(AclConfig.class); + List accessConfigs = new ArrayList<>(); + accessConfigs.add(accessConfig); + when(aclConfig.getPlainAccessConfigs()).thenReturn(accessConfigs); + when(plainPermissionManager.getAllAclConfig()).thenReturn(aclConfig); + when(authenticationMetadataManager.getUser(anyString())) + .thenReturn(CompletableFuture.completedFuture(null)); + try { + authMigrator.migrate(); + verify(authConfig, times(1)).isMigrateAuthFromV1Enabled(); + verify(plainPermissionManager, times(1)).getAllAclConfig(); + verify(authenticationMetadataManager, times(1)).createUser(any()); + verify(authorizationMetadataManager, never()).createAcl(any()); + } catch (final RuntimeException ex) { + assertEquals("Test Exception", ex.getMessage()); + } + } + + private PlainAccessConfig createPlainAccessConfig() { + PlainAccessConfig result = mock(PlainAccessConfig.class); + when(result.getAccessKey()).thenReturn("testAk"); + when(result.getSecretKey()).thenReturn("testSk"); + when(result.isAdmin()).thenReturn(false); + when(result.getTopicPerms()).thenReturn(new ArrayList<>()); + when(result.getGroupPerms()).thenReturn(new ArrayList<>()); + when(result.getDefaultTopicPerm()).thenReturn("PUB"); + when(result.getDefaultGroupPerm()).thenReturn(null); + return result; + } +} From 8b7f9cc5ef4ae40b8834df6dd0fd1f09a7f454bb Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Tue, 20 Aug 2024 11:09:07 +0800 Subject: [PATCH 112/265] [ISSUE #8534] Supports timer message queries (#8535) --- .../rocketmq/broker/topic/TopicConfigManager.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index d7c06180e9e..c71c65fe8bd 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; +import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.tieredstore.TieredMessageStore; import org.apache.rocketmq.tieredstore.metadata.MetadataStore; import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata; @@ -211,6 +212,17 @@ protected void init() { topicConfig.setWriteQueueNums(1); putTopicConfig(topicConfig); } + + { + if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) { + String topic = TimerMessageStore.TIMER_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + TopicValidator.addSystemTopic(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } } protected TopicConfig putTopicConfig(TopicConfig topicConfig) { From 18f5f28af5c908d2d9986bbd824f237be9baf5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Wed, 21 Aug 2024 10:57:08 +0800 Subject: [PATCH 113/265] Set specific permissions to trigger the retry mechanism (#8566) --- .github/workflows/bazel.yml | 3 +++ .github/workflows/maven.yaml | 4 ++++ .github/workflows/rerun-workflow.yml | 3 +++ 3 files changed, 10 insertions(+) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 73a49aa6a64..268a06a79fd 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -8,6 +8,9 @@ on: - develop - bazel +permissions: + actions: write + jobs: build: name: "bazel-compile (${{ matrix.os }})" diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 3291d993ea0..449f637894c 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -4,6 +4,10 @@ on: types: [opened, reopened, synchronize] push: branches: [master, develop, bazel] + +permissions: + actions: write + jobs: java_build: name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})" diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml index 2f3258c1f6e..bf83fc51b63 100644 --- a/.github/workflows/rerun-workflow.yml +++ b/.github/workflows/rerun-workflow.yml @@ -5,6 +5,9 @@ on: run_id: required: true +permissions: + actions: write + jobs: rerun: runs-on: ubuntu-latest From e2885acc52d4eea720e7aedc18067c9aef5c3425 Mon Sep 17 00:00:00 2001 From: syhleo <757187389@qq.com> Date: Wed, 21 Aug 2024 10:57:59 +0800 Subject: [PATCH 114/265] [ISSUE #8553] Add UnitTest of OffsetSerialize (#8554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ISSUE #8553] Add UnitTest of ZoneRouteRPCHook、OffsetSerialize --- .../RocksDBOffsetSerializeWrapperTest.java | 53 ++++++ .../route/ZoneRouteRPCHookMoreTest.java | 154 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java create mode 100644 namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java new file mode 100644 index 00000000000..dde0401e8ae --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.offset; + + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class RocksDBOffsetSerializeWrapperTest { + + private RocksDBOffsetSerializeWrapper wrapper; + + @Before + public void setUp() { + wrapper = new RocksDBOffsetSerializeWrapper(); + } + + @Test + public void testGetOffsetTable_ShouldReturnConcurrentHashMap() { + ConcurrentMap offsetTable = wrapper.getOffsetTable(); + assertNotNull("The offsetTable should not be null", offsetTable); + } + + @Test + public void testSetOffsetTable_ShouldSetTheOffsetTableCorrectly() { + ConcurrentMap newOffsetTable = new ConcurrentHashMap<>(); + wrapper.setOffsetTable(newOffsetTable); + ConcurrentMap offsetTable = wrapper.getOffsetTable(); + assertEquals("The offsetTable should be the same as the one set", newOffsetTable, offsetTable); + } + +} diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java new file mode 100644 index 00000000000..ed42becdf53 --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/route/ZoneRouteRPCHookMoreTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.route; + + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.QueueData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ZoneRouteRPCHookMoreTest { + + private ZoneRouteRPCHook zoneRouteRPCHook; + + @Before + public void setUp() { + zoneRouteRPCHook = new ZoneRouteRPCHook(); + } + + @Test + public void testFilterByZoneName_ValidInput_ShouldFilterCorrectly() { + // Arrange + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setBrokerDatas(generateBrokerDataList()); + topicRouteData.setQueueDatas(generateQueueDataList()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null); + request.setExtFields(createExtFields("true","ZoneA")); + + RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "remark"); + + // Act + zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response); + TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class); + + // Assert + assertNull(decodedResponse); + } + + @Test + public void testFilterByZoneName_NoZoneName_ShouldNotFilter() { + // Arrange + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setBrokerDatas(generateBrokerDataList()); + topicRouteData.setQueueDatas(generateQueueDataList()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null); + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, "true"); + request.setExtFields(extFields); + + RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); + + // Act + zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response); + TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class); + + // Assert + assertEquals(topicRouteData.getBrokerDatas().size(), 2); + assertEquals(topicRouteData.getQueueDatas().size(), 2); + } + + @Test + public void testFilterByZoneName_ZoneModeFalse_ShouldNotFilter() { + // Arrange + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setBrokerDatas(generateBrokerDataList()); + topicRouteData.setQueueDatas(generateQueueDataList()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC,null); + request.setExtFields(createExtFields("false","ZoneA")); + + RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS ,null); + + // Act + zoneRouteRPCHook.doAfterResponse("127.0.0.1", request, response); + TopicRouteData decodedResponse = RemotingSerializable.decode(response.getBody(), TopicRouteData.class); + + // Assert + assertEquals(topicRouteData.getBrokerDatas().size(), 2); + assertEquals(topicRouteData.getQueueDatas().size(), 2); + } + + private List generateBrokerDataList() { + List brokerDataList = new ArrayList<>(); + BrokerData brokerData1 = new BrokerData(); + brokerData1.setBrokerName("BrokerA"); + brokerData1.setZoneName("ZoneA"); + Map brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, "127.0.0.1:10911"); + brokerData1.setBrokerAddrs((HashMap) brokerAddrs); + brokerDataList.add(brokerData1); + + BrokerData brokerData2 = new BrokerData(); + brokerData2.setBrokerName("BrokerB"); + brokerData2.setZoneName("ZoneB"); + brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, "127.0.0.1:10912"); + brokerData2.setBrokerAddrs((HashMap) brokerAddrs); + brokerDataList.add(brokerData2); + + return brokerDataList; + } + + private List generateQueueDataList() { + List queueDataList = new ArrayList<>(); + QueueData queueData1 = new QueueData(); + queueData1.setBrokerName("BrokerA"); + queueDataList.add(queueData1); + + QueueData queueData2 = new QueueData(); + queueData2.setBrokerName("BrokerB"); + queueDataList.add(queueData2); + + return queueDataList; + } + + private HashMap createExtFields(String zoneMode, String zoneName) { + HashMap extFields = new HashMap<>(); + extFields.put(MixAll.ZONE_MODE, zoneMode); + extFields.put(MixAll.ZONE_NAME, zoneName); + return extFields; + } + +} \ No newline at end of file From e96e4456f92714dbaebcf8b3b56f7920b0f94830 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 21 Aug 2024 10:58:49 +0800 Subject: [PATCH 115/265] [ISSUE #8562] Add more test coverage for StatefulAuthorizationStrategy (#8563) * [ISSUE #8562] Add more test coverage for StatefulAuthorizationStrategy * Update * Update --- .../StatefulAuthorizationStrategyTest.java | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java new file mode 100644 index 00000000000..80e1f0b49e7 --- /dev/null +++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/strategy/StatefulAuthorizationStrategyTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.auth.authorization.strategy; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.commons.lang3.reflect.MethodUtils; +import org.apache.rocketmq.auth.authentication.model.Subject; +import org.apache.rocketmq.auth.authorization.context.AuthorizationContext; +import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext; +import org.apache.rocketmq.auth.authorization.exception.AuthorizationException; +import org.apache.rocketmq.auth.authorization.model.Resource; +import org.apache.rocketmq.auth.config.AuthConfig; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.action.Action; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class StatefulAuthorizationStrategyTest { + + @Mock + private AuthConfig authConfig; + + private StatefulAuthorizationStrategy statefulAuthorizationStrategy; + + @Before + public void setUp() { + when(authConfig.getStatefulAuthorizationCacheExpiredSecond()).thenReturn(60); + when(authConfig.getStatefulAuthorizationCacheMaxNum()).thenReturn(100); + Supplier metadataService = mock(Supplier.class); + statefulAuthorizationStrategy = spy(new StatefulAuthorizationStrategy(authConfig, metadataService)); + } + + @Test + public void testEvaluateChannelIdBlankDoesNotUseCache() { + AuthorizationContext context = mock(AuthorizationContext.class); + when(context.getChannelId()).thenReturn(null); + statefulAuthorizationStrategy.evaluate(context); + verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context); + } + + @Test + public void testEvaluateChannelIdNotNullCacheHit() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + DefaultAuthorizationContext context = new DefaultAuthorizationContext(); + context.setChannelId("channelId"); + context.setSubject(Subject.of("User")); + context.setResource(Resource.of("Cluster")); + context.setActions(new ArrayList<>()); + context.setSourceIp("sourceIp"); + Pair pair = Pair.of(true, null); + Cache> authCache = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.SECONDS) + .maximumSize(100) + .build(); + authCache.put(buildKey(context), pair); + statefulAuthorizationStrategy.authCache = authCache; + statefulAuthorizationStrategy.evaluate(context); + verify(statefulAuthorizationStrategy, never()).doEvaluate(context); + } + + @Test + public void testEvaluateChannelIdNotNullCacheMiss() { + DefaultAuthorizationContext context = new DefaultAuthorizationContext(); + context.setChannelId("channelId"); + context.setSubject(Subject.of("User")); + context.setResource(Resource.of("Cluster")); + context.setActions(Collections.singletonList(Action.PUB)); + context.setSourceIp("sourceIp"); + statefulAuthorizationStrategy.authCache = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.SECONDS) + .maximumSize(100) + .build(); + statefulAuthorizationStrategy.evaluate(context); + verify(statefulAuthorizationStrategy, times(1)).doEvaluate(context); + } + + @Test + public void testEvaluateChannelIdNotNullCacheException() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + DefaultAuthorizationContext context = new DefaultAuthorizationContext(); + context.setChannelId("channelId"); + context.setSubject(Subject.of("subjectKey")); + context.setResource(Resource.of("resourceKey")); + context.setActions(Collections.singletonList(Action.PUB)); + context.setSourceIp("sourceIp"); + AuthorizationException exception = new AuthorizationException("test"); + Pair pair = Pair.of(false, exception); + Cache> authCache = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.SECONDS) + .maximumSize(100) + .build(); + authCache.put(buildKey(context), pair); + statefulAuthorizationStrategy.authCache = authCache; + try { + statefulAuthorizationStrategy.evaluate(context); + fail("Expected AuthorizationException to be thrown"); + } catch (final AuthorizationException ex) { + assertEquals(exception, ex); + } + verify(statefulAuthorizationStrategy, never()).doEvaluate(context); + } + + private String buildKey(AuthorizationContext context) throws InvocationTargetException, NoSuchMethodException, IllegalAccessException { + return (String) MethodUtils.invokeMethod(statefulAuthorizationStrategy, true, "buildKey", context); + } +} From 09beca60eb5f2bd75fd0b092a1a3b0f583d010c5 Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Wed, 21 Aug 2024 13:44:26 +0800 Subject: [PATCH 116/265] [ISSUE #8549] Ipv6 enabled in broker, pickupStoreTimestamp size should be 20 (#8567) * fix: when ipv6 enabled in broker, pickupStoreTimestamp size should be 12(20-8) * fix: when ipv6 enabled in broker, pickupStoreTimestamp size should be 20 --- .../org/apache/rocketmq/store/DefaultMessageStore.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index a901e850ed6..f159c31a7be 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -58,6 +58,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.rocketmq.common.AbstractBrokerRunnable; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.BrokerConfig; @@ -1178,7 +1179,11 @@ public long getEarliestMessageTime() { if (this.getCommitLog() instanceof DLedgerCommitLog) { minPhyOffset += DLedgerEntry.BODY_OFFSET; } - final int size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 8; + int size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 8; + InetAddressValidator validator = InetAddressValidator.getInstance(); + if (validator.isValidInet6Address(this.brokerConfig.getBrokerIP1())) { + size = MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION + 20; + } return this.getCommitLog().pickupStoreTimestamp(minPhyOffset, size); } From 8859093a1d345dc98a119fd2ae6fc2b14faa76ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=98=9F=E7=81=BF?= <37405937+qianye1001@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:29:00 +0800 Subject: [PATCH 117/265] make ctx constructed in scheduleRenewTask (#8556) --- .../service/receipt/DefaultReceiptHandleManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java index 3948824a397..0cb519306eb 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java @@ -159,7 +159,8 @@ protected void scheduleRenewTask() { if (handle.getNextVisibleTime() - current > proxyConfig.getRenewAheadTimeMillis()) { return; } - renewalWorkerService.submit(() -> renewMessage(key, group, msgID, handleStr)); + renewalWorkerService.submit(() -> renewMessage(createContext("RenewMessage"), key, group, + msgID, handleStr)); }); } } catch (Exception e) { @@ -169,15 +170,15 @@ protected void scheduleRenewTask() { log.debug("scan for renewal done. cost:{}ms", stopwatch.elapsed().toMillis()); } - protected void renewMessage(ReceiptHandleGroupKey key, ReceiptHandleGroup group, String msgID, String handleStr) { + protected void renewMessage(ProxyContext context, ReceiptHandleGroupKey key, ReceiptHandleGroup group, String msgID, String handleStr) { try { - group.computeIfPresent(msgID, handleStr, messageReceiptHandle -> startRenewMessage(key, messageReceiptHandle)); + group.computeIfPresent(msgID, handleStr, messageReceiptHandle -> startRenewMessage(context, key, messageReceiptHandle)); } catch (Exception e) { log.error("error when renew message. msgID:{}, handleStr:{}", msgID, handleStr, e); } } - protected CompletableFuture startRenewMessage(ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle) { + protected CompletableFuture startRenewMessage(ProxyContext context, ReceiptHandleGroupKey key, MessageReceiptHandle messageReceiptHandle) { CompletableFuture resFuture = new CompletableFuture<>(); ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); long current = System.currentTimeMillis(); @@ -209,7 +210,6 @@ protected CompletableFuture startRenewMessage(ReceiptHandl } }); } else { - ProxyContext context = createContext("RenewMessage"); SubscriptionGroupConfig subscriptionGroupConfig = metadataService.getSubscriptionGroupConfig(context, messageReceiptHandle.getGroup()); if (subscriptionGroupConfig == null) { From 2a938fb1bcf35f18b2bcbf8616043be6fb105a36 Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Fri, 23 Aug 2024 11:18:03 +0800 Subject: [PATCH 118/265] [ISSUE #8058] Support for upgrading metadata in json to rocksdb (#8571) --- .github/workflows/maven.yaml | 27 +- .../broker}/RocksDBConfigManager.java | 60 +++- .../offset/RocksDBConsumerOffsetManager.java | 14 +- .../RocksDBSubscriptionGroupManager.java | 156 +++++++- .../SubscriptionGroupManager.java | 29 +- .../topic/RocksDBTopicConfigManager.java | 91 ++++- .../broker/topic/TopicConfigManager.java | 52 +-- .../RocksdbGroupConfigTransferTest.java | 340 ++++++++++++++++++ .../SubscriptionGroupManagerTest.java | 32 +- .../topic/RocksdbTopicConfigManagerTest.java | 15 +- .../topic/RocksdbTopicConfigTransferTest.java | 259 +++++++++++++ .../broker/topic/TopicConfigManagerTest.java | 10 +- .../apache/rocketmq/common/ConfigManager.java | 12 - .../common/config/AbstractRocksDBStorage.java | 21 +- .../common/config/ConfigRocksDBStorage.java | 39 +- .../store/config/MessageStoreConfig.java | 12 + .../tools/command/MQAdminStartup.java | 2 + .../metadata/RocksDBConfigToJsonCommand.java | 80 ++--- 18 files changed, 1069 insertions(+), 182 deletions(-) rename {common/src/main/java/org/apache/rocketmq/common/config => broker/src/main/java/org/apache/rocketmq/broker}/RocksDBConfigManager.java (63%) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 449f637894c..a49201b8a16 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -29,19 +29,28 @@ jobs: cache: "maven" - name: Build with Maven run: mvn -B package --file pom.xml - - name: Upload JVM crash logs + + - name: Run tests with increased memory and debug info + run: mvn test -X -Dparallel=none -DargLine="-Xmx1024m -XX:MaxPermSize=256m" + + - name: Upload Auth JVM crash logs if: failure() uses: actions/upload-artifact@v4 with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log retention-days: 1 - - name: Retry if failed - # if it failed , retry 2 times at most - if: failure() && fromJSON(github.run_attempt) < 3 - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for broker JVM crash logs + if: failure() run: | - echo "Attempting to retry workflow..." - gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file + echo "Checking for JVM crash logs..." + ls -al /Users/runner/work/rocketmq/rocketmq/broker/ + + - name: Upload broker JVM crash logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: jvm-crash-logs + path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log + retention-days: 1 \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java similarity index 63% rename from common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java index d1ec894685f..20358c4707f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/RocksDBConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java @@ -14,46 +14,66 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.common.config; +package org.apache.rocketmq.broker; +import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.config.ConfigRocksDBStorage; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.protocol.DataVersion; import org.rocksdb.FlushOptions; import org.rocksdb.RocksIterator; import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; +import java.nio.charset.StandardCharsets; import java.util.function.BiConsumer; public class RocksDBConfigManager { protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); - - protected volatile boolean isStop = false; - protected ConfigRocksDBStorage configRocksDBStorage = null; + public volatile boolean isStop = false; + public ConfigRocksDBStorage configRocksDBStorage = null; private FlushOptions flushOptions = null; private volatile long lastFlushMemTableMicroSecond = 0; + + private final String filePath; private final long memTableFlushInterval; + private DataVersion kvDataVersion = new DataVersion(); + - public RocksDBConfigManager(long memTableFlushInterval) { + public RocksDBConfigManager(String filePath, long memTableFlushInterval) { + this.filePath = filePath; this.memTableFlushInterval = memTableFlushInterval; } - public boolean load(String configFilePath, BiConsumer biConsumer) { + public boolean init() { this.isStop = false; - this.configRocksDBStorage = new ConfigRocksDBStorage(configFilePath); - if (!this.configRocksDBStorage.start()) { - return false; - } - RocksIterator iterator = this.configRocksDBStorage.iterator(); + this.configRocksDBStorage = new ConfigRocksDBStorage(filePath); + return this.configRocksDBStorage.start(); + } + public boolean loadDataVersion() { + String currDataVersionString = null; try { + byte[] dataVersion = this.configRocksDBStorage.getKvDataVersion(); + if (dataVersion != null && dataVersion.length > 0) { + currDataVersionString = new String(dataVersion, StandardCharsets.UTF_8); + } + kvDataVersion = StringUtils.isNotBlank(currDataVersionString) ? JSON.parseObject(currDataVersionString, DataVersion.class) : new DataVersion(); + return true; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public boolean loadData(BiConsumer biConsumer) { + try (RocksIterator iterator = this.configRocksDBStorage.iterator()) { iterator.seekToFirst(); while (iterator.isValid()) { biConsumer.accept(iterator.key(), iterator.value()); iterator.next(); } - } finally { - iterator.close(); } this.flushOptions = new FlushOptions(); @@ -103,6 +123,20 @@ public void delete(final byte[] keyBytes) throws Exception { this.configRocksDBStorage.delete(keyBytes); } + public void updateKvDataVersion() throws Exception { + kvDataVersion.nextVersion(); + this.configRocksDBStorage.updateKvDataVersion(JSON.toJSONString(kvDataVersion).getBytes(StandardCharsets.UTF_8)); + } + + public DataVersion getKvDataVersion() { + return kvDataVersion; + } + + public void updateForbidden(String key, String value) throws Exception { + this.configRocksDBStorage.updateForbidden(key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } + + public void batchPutWithWal(final WriteBatch batch) throws Exception { this.configRocksDBStorage.batchPutWithWal(batch); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java index 05b53b0bcf2..de293fc4992 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.utils.DataConverter; import org.rocksdb.WriteBatch; @@ -31,14 +31,19 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + protected RocksDBConfigManager rocksDBConfigManager; + public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(configFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override public boolean load() { - return this.rocksDBConfigManager.load(configFilePath(), this::decode0); + if (!rocksDBConfigManager.init()) { + return false; + } + return this.rocksDBConfigManager.loadData(this::decodeOffset); } @Override @@ -56,8 +61,7 @@ protected void removeConsumerOffset(String topicAtGroup) { } } - @Override - protected void decode0(final byte[] key, final byte[] body) { + protected void decodeOffset(final byte[] key, final byte[] body) { String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8); RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java index e9a81a8d686..7df72dbe686 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java @@ -16,39 +16,116 @@ */ package org.apache.rocketmq.broker.subscription; -import java.io.File; - +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.rocksdb.RocksIterator; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + protected RocksDBConfigManager rocksDBConfigManager; + public RocksDBSubscriptionGroupManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override public boolean load() { - if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) { + if (!rocksDBConfigManager.init()) { + return false; + } + if (!loadDataVersion() || !loadSubscriptionGroupAndForbidden()) { return false; } this.init(); return true; } + public boolean loadDataVersion() { + return this.rocksDBConfigManager.loadDataVersion(); + } + + public boolean loadSubscriptionGroupAndForbidden() { + return this.rocksDBConfigManager.loadData(this::decodeSubscriptionGroup) + && this.loadForbidden(this::decodeForbidden) + && merge(); + } + + public boolean loadForbidden(BiConsumer biConsumer) { + try (RocksIterator iterator = this.rocksDBConfigManager.configRocksDBStorage.forbiddenIterator()) { + iterator.seekToFirst(); + while (iterator.isValid()) { + biConsumer.accept(iterator.key(), iterator.value()); + iterator.next(); + } + } + return true; + } + + + private boolean merge() { + if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { + log.info("The switch is off, no merge operation is needed."); + return true; + } + if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { + log.info("json file and json back file not exist, so skip merge"); + return true; + } + + if (!super.load()) { + log.error("load group and forbidden info from json file error, startup will exit"); + return false; + } + + final ConcurrentMap groupTable = this.getSubscriptionGroupTable(); + final ConcurrentMap> forbiddenTable = this.getForbiddenTable(); + final DataVersion dataVersion = super.getDataVersion(); + final DataVersion kvDataVersion = this.getDataVersion(); + if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + for (Map.Entry entry : groupTable.entrySet()) { + putSubscriptionGroupConfig(entry.getValue()); + log.info("import subscription config to rocksdb, group={}", entry.getValue()); + } + for (Map.Entry> entry : forbiddenTable.entrySet()) { + try { + this.rocksDBConfigManager.updateForbidden(entry.getKey(), JSON.toJSONString(entry.getValue())); + log.info("import forbidden config to rocksdb, group={}", entry.getValue()); + } catch (Exception e) { + log.error("import forbidden config to rocksdb failed, group={}", entry.getValue()); + return false; + } + } + this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + updateDataVersion(); + } + log.info("finish marge subscription config from json file and merge to rocksdb"); + this.persist(); + + return true; + } + @Override public boolean stop() { return this.rocksDBConfigManager.stop(); } @Override - protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) { + public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) { String groupName = subscriptionGroupConfig.getGroupName(); SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); @@ -89,8 +166,8 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName return subscriptionGroupConfig; } - @Override - protected void decode0(byte[] key, byte[] body) { + + protected void decodeSubscriptionGroup(byte[] key, byte[] body) { String groupName = new String(key, DataConverter.CHARSET_UTF8); SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class); @@ -105,8 +182,63 @@ public synchronized void persist() { } } - @Override - public String configFilePath() { + public String rocksdbConfigFilePath() { return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "subscriptionGroups" + File.separator; } + + @Override + public DataVersion getDataVersion() { + return rocksDBConfigManager.getKvDataVersion(); + } + + @Override + public void updateDataVersion() { + try { + rocksDBConfigManager.updateKvDataVersion(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected void decodeForbidden(byte[] key, byte[] body) { + String forbiddenGroupName = new String(key, DataConverter.CHARSET_UTF8); + JSONObject jsonObject = JSON.parseObject(new String(body, DataConverter.CHARSET_UTF8)); + Set> entries = jsonObject.entrySet(); + ConcurrentMap forbiddenGroup = new ConcurrentHashMap<>(entries.size()); + for (Map.Entry entry : entries) { + forbiddenGroup.put(entry.getKey(), (Integer) entry.getValue()); + } + this.getForbiddenTable().put(forbiddenGroupName, forbiddenGroup); + log.info("load forbidden,{} value {}", forbiddenGroupName, forbiddenGroup.toString()); + } + + @Override + public void updateForbidden(String group, String topic, int forbiddenIndex, boolean setOrClear) { + try { + super.updateForbidden(group, topic, forbiddenIndex, setOrClear); + this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void setForbidden(String group, String topic, int forbiddenIndex) { + try { + super.setForbidden(group, topic, forbiddenIndex); + this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void clearForbidden(String group, String topic, int forbiddenIndex) { + try { + super.clearForbidden(group, topic, forbiddenIndex); + this.rocksDBConfigManager.updateForbidden(group, JSON.toJSONString(this.getForbiddenTable().get(group))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index e63b9305868..1d9614fe582 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -121,7 +121,7 @@ protected void init() { } } - protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) { + public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) { return this.subscriptionGroupTable.put(subscriptionGroupConfig.getGroupName(), subscriptionGroupConfig); } @@ -156,8 +156,7 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) log.info("create new subscription group, {}", config); } - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } @@ -214,7 +213,7 @@ public int getForbidden(String group, String topic) { return topicForbidden; } - private void updateForbiddenValue(String group, String topic, Integer forbidden) { + protected void updateForbiddenValue(String group, String topic, Integer forbidden) { if (forbidden == null || forbidden <= 0) { this.forbiddenTable.remove(group); log.info("clear group forbidden, {}@{} ", group, topic); @@ -233,8 +232,7 @@ private void updateForbiddenValue(String group, String topic, Integer forbidden) log.info("set group forbidden, {}@{} old: {} new: {}", group, topic, 0, forbidden); } - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } @@ -243,8 +241,7 @@ public void disableConsume(final String groupName) { SubscriptionGroupConfig old = getSubscriptionGroupConfig(groupName); if (old != null) { old.setConsumeEnable(false); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); } } @@ -261,8 +258,7 @@ public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) { if (null == preConfig) { log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString()); } - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } } @@ -331,8 +327,7 @@ public void deleteSubscriptionGroupConfig(final String groupName) { this.forbiddenTable.remove(groupName); if (old != null) { log.info("delete subscription group OK, subscription group:{}", old); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } else { log.warn("delete subscription group failed, subscription groupName: {} not exist", groupName); @@ -369,4 +364,14 @@ private Map current(String groupName) { } } } + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion.assignNewOne(dataVersion); + } + + public void updateDataVersion() { + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + dataVersion.nextVersion(stateMachineVersion); + } + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java index fddecf2d92a..2a89dd7e024 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java @@ -16,39 +16,86 @@ */ package org.apache.rocketmq.broker.topic; -import java.io.File; - +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.remoting.protocol.DataVersion; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; public class RocksDBTopicConfigManager extends TopicConfigManager { + protected RocksDBConfigManager rocksDBConfigManager; + public RocksDBTopicConfigManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override public boolean load() { - if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) { + if (!rocksDBConfigManager.init()) { + return false; + } + if (!loadDataVersion() || !loadTopicConfig()) { return false; } this.init(); return true; } + public boolean loadTopicConfig() { + return this.rocksDBConfigManager.loadData(this::decodeTopicConfig) && merge(); + } + + public boolean loadDataVersion() { + return this.rocksDBConfigManager.loadDataVersion(); + } + + private boolean merge() { + if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { + log.info("The switch is off, no merge operation is needed."); + return true; + } + if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { + log.info("json file and json back file not exist, so skip merge"); + return true; + } + + if (!super.load()) { + log.error("load topic config from json file error, startup will exit"); + return false; + } + + final ConcurrentMap topicConfigTable = this.getTopicConfigTable(); + final DataVersion dataVersion = super.getDataVersion(); + final DataVersion kvDataVersion = this.getDataVersion(); + if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + for (Map.Entry entry : topicConfigTable.entrySet()) { + putTopicConfig(entry.getValue()); + log.info("import topic config to rocksdb, topic={}", entry.getValue()); + } + this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + updateDataVersion(); + } + log.info("finish read topic config from json file and merge to rocksdb"); + this.persist(); + return true; + } + + @Override public boolean stop() { return this.rocksDBConfigManager.stop(); } - @Override - protected void decode0(byte[] key, byte[] body) { + protected void decodeTopicConfig(byte[] key, byte[] body) { String topicName = new String(key, DataConverter.CHARSET_UTF8); TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class); @@ -57,12 +104,7 @@ protected void decode0(byte[] key, byte[] body) { } @Override - public String configFilePath() { - return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "topics" + File.separator; - } - - @Override - protected TopicConfig putTopicConfig(TopicConfig topicConfig) { + public TopicConfig putTopicConfig(TopicConfig topicConfig) { String topicName = topicConfig.getTopicName(); TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig); try { @@ -92,4 +134,23 @@ public synchronized void persist() { this.rocksDBConfigManager.flushWAL(); } } + + public String rocksdbConfigFilePath() { + return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "topics" + File.separator; + } + + + @Override + public DataVersion getDataVersion() { + return rocksDBConfigManager.getKvDataVersion(); + } + + @Override + public void updateDataVersion() { + try { + rocksDBConfigManager.updateKvDataVersion(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index c71c65fe8bd..eab2896b001 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -225,7 +225,7 @@ protected void init() { } } - protected TopicConfig putTopicConfig(TopicConfig topicConfig) { + public TopicConfig putTopicConfig(TopicConfig topicConfig) { return this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } @@ -293,8 +293,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri putTopicConfig(topicConfig); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); createNew = true; @@ -337,8 +336,7 @@ public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register } log.info("Create new topic [{}] config:[{}]", topicConfig.getTopicName(), topicConfig); putTopicConfig(topicConfig); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); createNew = true; this.persist(); } finally { @@ -397,8 +395,7 @@ public TopicConfig createTopicInSendMessageBackMethod( log.info("create new topic {}", topicConfig); putTopicConfig(topicConfig); createNew = true; - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } finally { this.topicConfigTableLock.unlock(); @@ -438,8 +435,7 @@ public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQue log.info("create new topic {}", topicConfig); putTopicConfig(topicConfig); createNew = true; - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } finally { this.topicConfigTableLock.unlock(); @@ -472,8 +468,7 @@ public void updateTopicUnitFlag(final String topic, final boolean unit) { putTopicConfig(topicConfig); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); registerBrokerData(topicConfig); @@ -495,8 +490,7 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) putTopicConfig(topicConfig); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); registerBrokerData(topicConfig); @@ -509,7 +503,6 @@ private void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig Map newAttributes = request(topicConfig); Map currentAttributes = current(topicConfig.getTopicName()); - Map finalAttributes = AttributeUtil.alterCurrentAttributes( this.topicConfigTable.get(topicConfig.getTopicName()) == null, TopicAttributes.ALL, @@ -526,8 +519,7 @@ private void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig log.info("create new topic [{}]", topicConfig); } - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); } public void updateTopicConfig(final TopicConfig topicConfig) { @@ -581,25 +573,8 @@ public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) { } } - // We don't have a mandatory rule to maintain the validity of order conf in NameServer, - // so we may overwrite the order field mistakenly. - // To avoid the above case, we comment the below codes, please use mqadmin API to update - // the order filed. - /*for (Map.Entry entry : this.topicConfigTable.entrySet()) { - String topic = entry.getKey(); - if (!orderTopics.contains(topic)) { - TopicConfig topicConfig = entry.getValue(); - if (topicConfig.isOrder()) { - topicConfig.setOrder(false); - isChange = true; - log.info("update order topic config, topic={}, order={}", topic, false); - } - } - }*/ - if (isChange) { - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } } @@ -623,8 +598,7 @@ public void deleteTopicConfig(final String topic) { TopicConfig old = removeTopicConfig(topic); if (old != null) { log.info("delete topic config OK, topic: {}", old); - long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - dataVersion.nextVersion(stateMachineVersion); + updateDataVersion(); this.persist(); } else { log.warn("delete topic config failed, topic: {} not exists", topic); @@ -739,5 +713,11 @@ public boolean containsTopic(String topic) { return topicConfigTable.containsKey(topic); } + public void updateDataVersion() { + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + dataVersion.nextVersion(stateMachineVersion); + } + + } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java new file mode 100644 index 00000000000..205e642843b --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.subscription; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RocksdbGroupConfigTransferTest { + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); + + private RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager; + + private SubscriptionGroupManager jsonSubscriptionGroupManager; + @Mock + private BrokerController brokerController; + + @Mock + private DefaultMessageStore defaultMessageStore; + + @Before + public void init() { + if (notToBeExecuted()) { + return; + } + BrokerConfig brokerConfig = new BrokerConfig(); + Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + messageStoreConfig.setTransferMetadataJsonToRocksdb(true); + Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); + when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); + } + + @After + public void destroy() { + if (notToBeExecuted()) { + return; + } + Path pathToBeDeleted = Paths.get(basePath); + + try { + Files.walk(pathToBeDeleted) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + // ignore + } + }); + } catch (IOException e) { + // ignore + } + if (rocksDBSubscriptionGroupManager != null) { + rocksDBSubscriptionGroupManager.stop(); + } + } + + + public void initRocksDBSubscriptionGroupManager() { + if (rocksDBSubscriptionGroupManager == null) { + rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController); + rocksDBSubscriptionGroupManager.load(); + } + } + + public void initJsonSubscriptionGroupManager() { + if (jsonSubscriptionGroupManager == null) { + jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController); + jsonSubscriptionGroupManager.load(); + } + } + + @Test + public void theFirstTimeLoadJsonSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initJsonSubscriptionGroupManager(); + DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(0L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size()); + } + + @Test + public void theFirstTimeLoadRocksDBSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initRocksDBSubscriptionGroupManager(); + DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(0L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + } + + + @Test + public void addGroupLoadJsonSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initJsonSubscriptionGroupManager(); + int beforeSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size(); + String groupName = "testAddGroupConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(groupName); + subscriptionGroupConfig.setAttributes(attributes); + DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + jsonSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig); + + int afterSize = jsonSubscriptionGroupManager.getSubscriptionGroupTable().size(); + DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + + Assert.assertEquals(0, beforeDataVersionCounter); + Assert.assertEquals(1, afterDataVersionCounter); + Assert.assertEquals(1, afterSize - beforeSize); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + } + + @Test + public void addForbiddenGroupLoadJsonSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initJsonSubscriptionGroupManager(); + int beforeSize = jsonSubscriptionGroupManager.getForbiddenTable().size(); + String groupName = "testAddGroupConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(groupName); + subscriptionGroupConfig.setAttributes(attributes); + DataVersion beforeDataVersion = jsonSubscriptionGroupManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + jsonSubscriptionGroupManager.setForbidden(groupName, "topic", 0); + int afterSize = jsonSubscriptionGroupManager.getForbiddenTable().size(); + DataVersion afterDataVersion = jsonSubscriptionGroupManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + + Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter); + Assert.assertEquals(1, afterSize - beforeSize); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + } + + @Test + public void addGroupLoadRocksdbSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initRocksDBSubscriptionGroupManager(); + int beforeSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(); + String groupName = "testAddGroupConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(groupName); + subscriptionGroupConfig.setAttributes(attributes); + DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + rocksDBSubscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig); + int afterSize = rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(); + DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + Assert.assertEquals(1, afterDataVersionCounter); + Assert.assertEquals(0, beforeDataVersionCounter); + Assert.assertEquals(1, afterSize - beforeSize); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + } + + @Test + public void addForbiddenLoadRocksdbSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + initRocksDBSubscriptionGroupManager(); + int beforeSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size(); + String groupName = "testAddGroupConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(groupName); + subscriptionGroupConfig.setAttributes(attributes); + DataVersion beforeDataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + rocksDBSubscriptionGroupManager.updateForbidden(groupName, "topic", 0, true); + + int afterSize = rocksDBSubscriptionGroupManager.getForbiddenTable().size(); + DataVersion afterDataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + Assert.assertEquals(1, afterDataVersionCounter - beforeDataVersionCounter); + Assert.assertEquals(1, afterSize - beforeSize); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + } + + @Test + public void theSecondTimeLoadJsonSubscriptionGroupManager() { + if (notToBeExecuted()) { + return; + } + addGroupLoadJsonSubscriptionGroupManager(); + jsonSubscriptionGroupManager.stop(); + rocksDBSubscriptionGroupManager = null; + addForbiddenGroupLoadJsonSubscriptionGroupManager(); + jsonSubscriptionGroupManager.stop(); + rocksDBSubscriptionGroupManager = null; + jsonSubscriptionGroupManager = new SubscriptionGroupManager(brokerController); + jsonSubscriptionGroupManager.load(); + DataVersion dataVersion = jsonSubscriptionGroupManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(2L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getForbiddenTable().size()); + Assert.assertNotEquals(0, jsonSubscriptionGroupManager.getSubscriptionGroupTable().size()); + } + + @Test + public void theSecondTimeLoadRocksdbTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + addGroupLoadRocksdbSubscriptionGroupManager(); + rocksDBSubscriptionGroupManager.stop(); + rocksDBSubscriptionGroupManager = null; + addForbiddenLoadRocksdbSubscriptionGroupManager(); + rocksDBSubscriptionGroupManager.stop(); + rocksDBSubscriptionGroupManager = null; + rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController); + rocksDBSubscriptionGroupManager.load(); + DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(2L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + } + + + @Test + public void jsonUpgradeToRocksdb() { + if (notToBeExecuted()) { + return; + } + addGroupLoadJsonSubscriptionGroupManager(); + addForbiddenGroupLoadJsonSubscriptionGroupManager(); + initRocksDBSubscriptionGroupManager(); + DataVersion dataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(3L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size()); + + rocksDBSubscriptionGroupManager.stop(); + rocksDBSubscriptionGroupManager = new RocksDBSubscriptionGroupManager(brokerController); + rocksDBSubscriptionGroupManager.load(); + dataVersion = rocksDBSubscriptionGroupManager.getDataVersion(); + Assert.assertEquals(3L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getForbiddenTable().size()); + Assert.assertNotEquals(0, rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertEquals(rocksDBSubscriptionGroupManager.getSubscriptionGroupTable().size(), jsonSubscriptionGroupManager.getSubscriptionGroupTable().size()); + Assert.assertEquals(rocksDBSubscriptionGroupManager.getForbiddenTable().size(), jsonSubscriptionGroupManager.getForbiddenTable().size()); + } + + private boolean notToBeExecuted() { + return MixAll.isMac(); + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java index 3c829437cf1..3ed4ac11a40 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java @@ -18,7 +18,11 @@ package org.apache.rocketmq.broker.subscription; import com.google.common.collect.ImmutableMap; + +import java.nio.file.Paths; import java.util.Map; +import java.util.UUID; + import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.SubscriptionGroupAttributes; @@ -30,36 +34,42 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class SubscriptionGroupManagerTest { private String group = "group"; + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); @Mock private BrokerController brokerControllerMock; private SubscriptionGroupManager subscriptionGroupManager; @Before public void before() { + if (notToBeExecuted()) { + return; + } SubscriptionGroupAttributes.ALL.put("test", new BooleanAttribute( "test", false, false )); subscriptionGroupManager = spy(new SubscriptionGroupManager(brokerControllerMock)); - when(brokerControllerMock.getMessageStore()).thenReturn(null); - doNothing().when(subscriptionGroupManager).persist(); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + Mockito.lenient().when(brokerControllerMock.getMessageStoreConfig()).thenReturn(messageStoreConfig); } @After public void destroy() { - if (MixAll.isMac()) { + if (notToBeExecuted()) { return; } if (subscriptionGroupManager != null) { @@ -69,18 +79,18 @@ public void destroy() { @Test public void testUpdateAndCreateSubscriptionGroupInRocksdb() { - if (MixAll.isMac()) { + if (notToBeExecuted()) { return; } - when(brokerControllerMock.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); - subscriptionGroupManager = spy(new RocksDBSubscriptionGroupManager(brokerControllerMock)); - subscriptionGroupManager.load(); group += System.currentTimeMillis(); updateSubscriptionGroupConfig(); } @Test public void updateSubscriptionGroupConfig() { + if (notToBeExecuted()) { + return; + } SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); subscriptionGroupConfig.setGroupName(group); Map attr = ImmutableMap.of("+test", "true"); @@ -99,4 +109,8 @@ public void updateSubscriptionGroupConfig() { assertThatThrownBy(() -> subscriptionGroupManager.updateSubscriptionGroupConfig(subscriptionGroupConfig1)) .isInstanceOf(RuntimeException.class).hasMessage("attempt to update an unchangeable attribute. key: test"); } + + private boolean notToBeExecuted() { + return MixAll.isMac(); + } } \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java index ed71a3313a8..b0e0d057363 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java @@ -16,10 +16,13 @@ */ package org.apache.rocketmq.broker.topic; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; + import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; @@ -39,6 +42,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import static com.google.common.collect.Sets.newHashSet; @@ -47,6 +51,10 @@ @RunWith(MockitoJUnitRunner.class) public class RocksdbTopicConfigManagerTest { + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); + private RocksDBTopicConfigManager topicConfigManager; @Mock private BrokerController brokerController; @@ -62,9 +70,11 @@ public void init() { BrokerConfig brokerConfig = new BrokerConfig(); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + messageStoreConfig.setTransferMetadataJsonToRocksdb(true); when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); - when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); + Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); + Mockito.lenient().when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); topicConfigManager = new RocksDBTopicConfigManager(brokerController); topicConfigManager.load(); } @@ -197,7 +207,6 @@ public void testNormalAddKeyOnCreating() { TopicConfig existingTopicConfig = topicConfigManager.getTopicConfigTable().get(topic); Assert.assertEquals("enum-2", existingTopicConfig.getAttributes().get("enum.key")); Assert.assertEquals("16", existingTopicConfig.getAttributes().get("long.range.key")); - // assert file } @Test diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java new file mode 100644 index 00000000000..2a727090987 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.topic; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RocksdbTopicConfigTransferTest { + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); + + private RocksDBTopicConfigManager rocksdbTopicConfigManager; + + private TopicConfigManager jsonTopicConfigManager; + @Mock + private BrokerController brokerController; + + @Mock + private DefaultMessageStore defaultMessageStore; + + @Before + public void init() { + if (notToBeExecuted()) { + return; + } + BrokerConfig brokerConfig = new BrokerConfig(); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + messageStoreConfig.setTransferMetadataJsonToRocksdb(true); + Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); + when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); + } + + @After + public void destroy() { + if (notToBeExecuted()) { + return; + } + Path pathToBeDeleted = Paths.get(basePath); + try { + Files.walk(pathToBeDeleted) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + // ignore + } + }); + } catch (IOException e) { + // ignore + } + if (rocksdbTopicConfigManager != null) { + rocksdbTopicConfigManager.stop(); + } + } + + public void initRocksdbTopicConfigManager() { + if (rocksdbTopicConfigManager == null) { + rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController); + rocksdbTopicConfigManager.load(); + } + } + + public void initJsonTopicConfigManager() { + if (jsonTopicConfigManager == null) { + jsonTopicConfigManager = new TopicConfigManager(brokerController); + jsonTopicConfigManager.load(); + } + } + + @Test + public void theFirstTimeLoadJsonTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + initJsonTopicConfigManager(); + DataVersion dataVersion = jsonTopicConfigManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(0L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size()); + } + + @Test + public void theFirstTimeLoadRocksdbTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + initRocksdbTopicConfigManager(); + DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(0L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size()); + } + + + @Test + public void addTopicLoadJsonTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + initJsonTopicConfigManager(); + String topicName = "testAddTopicConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setTopicName(topicName); + topicConfig.setAttributes(attributes); + DataVersion beforeDataVersion = jsonTopicConfigManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + jsonTopicConfigManager.updateTopicConfig(topicConfig); + + DataVersion afterDataVersion = jsonTopicConfigManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + + Assert.assertEquals(0, beforeDataVersionCounter); + Assert.assertEquals(1, afterDataVersionCounter); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + } + + @Test + public void addTopicLoadRocksdbTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + initRocksdbTopicConfigManager(); + String topicName = "testAddTopicConfig-" + System.currentTimeMillis(); + + Map attributes = new HashMap<>(); + + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setTopicName(topicName); + topicConfig.setAttributes(attributes); + DataVersion beforeDataVersion = rocksdbTopicConfigManager.getDataVersion(); + long beforeDataVersionCounter = beforeDataVersion.getCounter().get(); + long beforeTimestamp = beforeDataVersion.getTimestamp(); + + rocksdbTopicConfigManager.updateTopicConfig(topicConfig); + + DataVersion afterDataVersion = rocksdbTopicConfigManager.getDataVersion(); + long afterDataVersionCounter = afterDataVersion.getCounter().get(); + long afterTimestamp = afterDataVersion.getTimestamp(); + Assert.assertEquals(0, beforeDataVersionCounter); + Assert.assertEquals(1, afterDataVersionCounter); + Assert.assertTrue(afterTimestamp >= beforeTimestamp); + } + + @Test + public void theSecondTimeLoadJsonTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + addTopicLoadJsonTopicConfigManager(); + jsonTopicConfigManager.stop(); + jsonTopicConfigManager = new TopicConfigManager(brokerController); + jsonTopicConfigManager.load(); + DataVersion dataVersion = jsonTopicConfigManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(1L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, jsonTopicConfigManager.getTopicConfigTable().size()); + } + + @Test + public void theSecondTimeLoadRocksdbTopicConfigManager() { + if (notToBeExecuted()) { + return; + } + addTopicLoadRocksdbTopicConfigManager(); + rocksdbTopicConfigManager.stop(); + rocksdbTopicConfigManager = null; + rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController); + rocksdbTopicConfigManager.load(); + DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(1L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size()); + } + + @Test + public void jsonUpgradeToRocksdb() { + if (notToBeExecuted()) { + return; + } + addTopicLoadJsonTopicConfigManager(); + initRocksdbTopicConfigManager(); + DataVersion dataVersion = rocksdbTopicConfigManager.getDataVersion(); + Assert.assertNotNull(dataVersion); + Assert.assertEquals(2L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size()); + Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), jsonTopicConfigManager.getTopicConfigTable().size()); + + rocksdbTopicConfigManager.stop(); + rocksdbTopicConfigManager = new RocksDBTopicConfigManager(brokerController); + rocksdbTopicConfigManager.load(); + dataVersion = rocksdbTopicConfigManager.getDataVersion(); + Assert.assertEquals(2L, dataVersion.getCounter().get()); + Assert.assertEquals(0L, dataVersion.getStateVersion()); + Assert.assertNotEquals(0, rocksdbTopicConfigManager.getTopicConfigTable().size()); + Assert.assertEquals(rocksdbTopicConfigManager.getTopicConfigTable().size(), rocksdbTopicConfigManager.getTopicConfigTable().size()); + } + + + private boolean notToBeExecuted() { + return MixAll.isMac(); + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java index 6052a79d413..3fd1d14c3a2 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java @@ -16,10 +16,13 @@ */ package org.apache.rocketmq.broker.topic; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; + import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicAttributes; @@ -37,6 +40,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import static com.google.common.collect.Sets.newHashSet; @@ -45,6 +49,9 @@ @RunWith(MockitoJUnitRunner.class) public class TopicConfigManagerTest { + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); private TopicConfigManager topicConfigManager; @Mock private BrokerController brokerController; @@ -57,8 +64,9 @@ public void init() { BrokerConfig brokerConfig = new BrokerConfig(); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); + Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); topicConfigManager = new TopicConfigManager(brokerController); } diff --git a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java index 099f0d8d560..3fcf466fd77 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java +++ b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java @@ -16,11 +16,9 @@ */ package org.apache.rocketmq.common; -import org.apache.rocketmq.common.config.RocksDBConfigManager; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.rocksdb.Statistics; import java.io.IOException; import java.util.Map; @@ -28,8 +26,6 @@ public abstract class ConfigManager { private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); - protected RocksDBConfigManager rocksDBConfigManager; - public boolean load() { String fileName = null; try { @@ -89,10 +85,6 @@ public synchronized void persist() { } } - protected void decode0(final byte[] key, final byte[] body) { - - } - public boolean stop() { return true; } @@ -104,8 +96,4 @@ public boolean stop() { public abstract String encode(final boolean prettyFormat); public abstract void decode(final String jsonString); - - public Statistics getStatistics() { - return rocksDBConfigManager == null ? null : rocksDBConfigManager.getStatistics(); - } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index ed3a12dc245..f88b8e198bf 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -16,18 +16,7 @@ */ package org.apache.rocketmq.common.config; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - import com.google.common.collect.Maps; - import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; @@ -51,6 +40,16 @@ import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + import static org.rocksdb.RocksDB.NOT_FOUND; public abstract class AbstractRocksDBStorage { diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index b40f8046e84..f657d9cf2d2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -18,6 +18,7 @@ import java.io.File; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -54,6 +55,15 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { + private static final byte[] KV_DATA_VERSION_COLUMN_FAMILY_NAME = "kvDataVersion".getBytes(StandardCharsets.UTF_8); + private static final byte[] KV_DATA_VERSION_KEY = "kvDataVersionKey".getBytes(StandardCharsets.UTF_8); + protected ColumnFamilyHandle kvDataVersionFamilyHandle; + + private static final byte[] FORBIDDEN_COLUMN_FAMILY_NAME = "forbidden".getBytes(StandardCharsets.UTF_8); + protected ColumnFamilyHandle forbiddenFamilyHandle; + + + public ConfigRocksDBStorage(final String dbPath) { super(); this.dbPath = dbPath; @@ -115,11 +125,15 @@ protected boolean postLoad() { ColumnFamilyOptions defaultOptions = createConfigOptions(); this.cfOptions.add(defaultOptions); cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); - + cfDescriptors.add(new ColumnFamilyDescriptor(KV_DATA_VERSION_COLUMN_FAMILY_NAME, defaultOptions)); + cfDescriptors.add(new ColumnFamilyDescriptor(FORBIDDEN_COLUMN_FAMILY_NAME, defaultOptions)); final List cfHandles = new ArrayList(); open(cfDescriptors, cfHandles); this.defaultCFHandle = cfHandles.get(0); + this.kvDataVersionFamilyHandle = cfHandles.get(1); + this.forbiddenFamilyHandle = cfHandles.get(2); + } catch (final Exception e) { AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e); return false; @@ -129,7 +143,8 @@ protected boolean postLoad() { @Override protected void preShutdown() { - + this.kvDataVersionFamilyHandle.close(); + this.forbiddenFamilyHandle.close(); } private ColumnFamilyOptions createConfigOptions() { @@ -225,6 +240,22 @@ public byte[] get(final byte[] keyBytes) throws Exception { return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes); } + public void updateKvDataVersion(final byte[] valueBytes) throws Exception { + put(this.kvDataVersionFamilyHandle, this.ableWalWriteOptions, KV_DATA_VERSION_KEY, KV_DATA_VERSION_KEY.length, valueBytes, valueBytes.length); + } + + public byte[] getKvDataVersion() throws Exception { + return get(this.kvDataVersionFamilyHandle, this.totalOrderReadOptions, KV_DATA_VERSION_KEY); + } + + public void updateForbidden(final byte[] keyBytes, final byte[] valueBytes) throws Exception { + put(this.forbiddenFamilyHandle, this.ableWalWriteOptions, keyBytes, keyBytes.length, valueBytes, valueBytes.length); + } + + public byte[] getForbidden(final byte[] keyBytes) throws Exception { + return get(this.forbiddenFamilyHandle, this.totalOrderReadOptions, keyBytes); + } + public void delete(final byte[] keyBytes) throws Exception { delete(this.defaultCFHandle, this.ableWalWriteOptions, keyBytes); } @@ -246,6 +277,10 @@ public RocksIterator iterator() { return this.db.newIterator(this.defaultCFHandle, this.totalOrderReadOptions); } + public RocksIterator forbiddenIterator() { + return this.db.newIterator(this.forbiddenFamilyHandle, this.totalOrderReadOptions); + } + public void rangeDelete(final byte[] startKey, final byte[] endKey) throws RocksDBException { rangeDelete(this.defaultCFHandle, this.writeOptions, startKey, endKey); } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 5b2a1931b3b..0b45d92418e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -105,6 +105,9 @@ public class MessageStoreConfig { // default, defaultRocksDB @ImportantField private String storeType = StoreType.DEFAULT.getStoreType(); + + private boolean transferMetadataJsonToRocksdb = false; + // ConsumeQueue file size,default is 30W private int mappedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE; // enable consume queue ext @@ -1842,4 +1845,13 @@ public boolean isPutConsumeQueueDataByFileChannel() { public void setPutConsumeQueueDataByFileChannel(boolean putConsumeQueueDataByFileChannel) { this.putConsumeQueueDataByFileChannel = putConsumeQueueDataByFileChannel; } + + public boolean isTransferMetadataJsonToRocksdb() { + return transferMetadataJsonToRocksdb; + } + + public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocksdb) { + this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; + } + } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index e785934ba37..d56ed053268 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -92,6 +92,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; import org.apache.rocketmq.tools.command.message.SendMessageCommand; +import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; @@ -300,6 +301,7 @@ public static void initCommand() { initCommand(new GetAclSubCommand()); initCommand(new ListAclSubCommand()); initCommand(new CopyAclsSubCommand()); + initCommand(new RocksDBConfigToJsonCommand()); } private static void printHelp() { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java index b987ad873be..1d81287ac7d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java @@ -14,18 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.rocketmq.tools.command.metadata; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.config.RocksDBConfigManager; +import org.apache.rocketmq.common.config.ConfigRocksDBStorage; import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; +import org.rocksdb.RocksIterator; import java.io.File; import java.util.HashMap; @@ -47,8 +50,8 @@ public String commandDesc() { @Override public Options buildCommandlineOptions(Options options) { - Option pathOption = new Option("p", "path", true, - "Absolute path to the metadata directory"); + Option pathOption = new Option("p", "configPath", true, + "Absolute path to the metadata config directory"); pathOption.setRequired(true); options.addOption(pathOption); @@ -62,57 +65,50 @@ public Options buildCommandlineOptions(Options options) { @Override public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { - String path = commandLine.getOptionValue("path").trim(); + String path = commandLine.getOptionValue("configPath").trim(); if (StringUtils.isEmpty(path) || !new File(path).exists()) { System.out.print("Rocksdb path is invalid.\n"); return; } String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + if (!path.endsWith("/")) { + path += "/"; + } + path += configType; + + ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true); + configRocksDBStorage.start(); + RocksIterator iterator = configRocksDBStorage.iterator(); - final long memTableFlushInterval = 60 * 60 * 1000L; - RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); try { - if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { - // for topics.json - final Map topicsJsonConfig = new HashMap<>(); - final Map topicConfigTable = new HashMap<>(); - boolean isLoad = kvConfigManager.load(path, (key, value) -> { - final String topic = new String(key, DataConverter.CHARSET_UTF8); - final String topicConfig = new String(value, DataConverter.CHARSET_UTF8); - final JSONObject jsonObject = JSONObject.parseObject(topicConfig); - topicConfigTable.put(topic, jsonObject); - }); + final Map configMap = new HashMap<>(); + final Map configTable = new HashMap<>(); + iterator.seekToFirst(); + while (iterator.isValid()) { + final byte[] key = iterator.key(); + final byte[] value = iterator.value(); + final String name = new String(key, DataConverter.CHARSET_UTF8); + final String config = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(config); + configTable.put(name, jsonObject); + iterator.next(); + } + byte[] kvDataVersion = configRocksDBStorage.getKvDataVersion(); + configMap.put("dataVersion", + JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8))); - if (isLoad) { - topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); - final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); - System.out.print(topicsJsonStr + "\n"); - return; - } + if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { + configMap.put("topicConfigTable", JSON.parseObject(JSONObject.toJSONString(configTable))); } if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { - // for subscriptionGroup.json - final Map subscriptionGroupJsonConfig = new HashMap<>(); - final Map subscriptionGroupTable = new HashMap<>(); - boolean isLoad = kvConfigManager.load(path, (key, value) -> { - final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8); - final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8); - final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); - subscriptionGroupTable.put(subscriptionGroup, jsonObject); - }); - - if (isLoad) { - subscriptionGroupJsonConfig.put("subscriptionGroupTable", - (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); - final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); - System.out.print(subscriptionGroupJsonStr + "\n"); - return; - } + configMap.put("subscriptionGroupTable", JSON.parseObject(JSONObject.toJSONString(configTable))); } - System.out.print("Config type was not recognized, configType=" + configType + "\n"); + System.out.print(JSONObject.toJSONString(configMap, true) + "\n"); + } catch (Exception e) { + System.out.print("Error occurred while converting RocksDB kv config to json, " + "configType=" + configType + ", " + e.getMessage() + "\n"); } finally { - kvConfigManager.stop(); + configRocksDBStorage.shutdown(); } } -} +} \ No newline at end of file From 7444aa28967565a1a113678f9d726201528de6ee Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 23 Aug 2024 11:43:50 +0800 Subject: [PATCH 119/265] [ISSUE #8573] Correct mismatched comments (#8574) * [ISSUE #8573] Correct mismatched comments * Update * Update --- .../rocketmq/client/consumer/DefaultMQPushConsumer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 2d9fb73cec4..94785c69708 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -338,7 +338,7 @@ public DefaultMQPushConsumer(final String consumerGroup, boolean enableMsgTrace, /** * Constructor specifying consumer group, RPC hook and message queue allocating algorithm. * - * @param consumerGroup Consume queue. + * @param consumerGroup Consumer group. * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy Message queue allocating algorithm. */ @@ -350,7 +350,7 @@ public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, /** * Constructor specifying consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name. * - * @param consumerGroup Consume queue. + * @param consumerGroup Consumer group. * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy message queue allocating algorithm. * @param enableMsgTrace Switch flag instance for message trace. @@ -394,7 +394,7 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup, * Constructor specifying namespace, consumer group, RPC hook and message queue allocating algorithm. * * @param namespace Namespace for this MQ Producer instance. - * @param consumerGroup Consume queue. + * @param consumerGroup Consumer group. * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy Message queue allocating algorithm. */ @@ -412,7 +412,7 @@ public DefaultMQPushConsumer(final String namespace, final String consumerGroup, * Constructor specifying namespace, consumer group, RPC hook, message queue allocating algorithm, enabled msg trace flag and customized trace topic name. * * @param namespace Namespace for this MQ Producer instance. - * @param consumerGroup Consume queue. + * @param consumerGroup Consumer group. * @param rpcHook RPC hook to execute before each remoting command. * @param allocateMessageQueueStrategy message queue allocating algorithm. * @param enableMsgTrace Switch flag instance for message trace. From e8d87b195d6c639ed376feaa585e623c4a4e4f97 Mon Sep 17 00:00:00 2001 From: maclong1989 <814742806@qq.com> Date: Sat, 24 Aug 2024 17:52:13 +0800 Subject: [PATCH 120/265] Fix document typo in SlaveActingMasterMode.md (#8575) Signed-off-by: maclong1989 <814742806@qq.com> --- docs/cn/SlaveActingMasterMode.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/SlaveActingMasterMode.md b/docs/cn/SlaveActingMasterMode.md index b08cf0d9af3..b64adc61204 100644 --- a/docs/cn/SlaveActingMasterMode.md +++ b/docs/cn/SlaveActingMasterMode.md @@ -80,7 +80,7 @@ Slave Broker发现自己是该组中最小的brokerId，将会开启代理模式 代理模式开启后，brokerId最小的Slave会承担起二级消息的扫描和重新投递功能。 -二级消息一般分为两个阶段，发送或者消费时会发送到一个特殊topic中，后台会有线程会扫描，最终的满足要求的消息会被重新投递到Commitlog中。我们可以让brokerId最小的Slave进行扫描，但如果扫描之后的消息重新投递到本Commitlog，那将会破坏Slave不可写的语义，造成Commitlog分叉。因此RoccketMQ 5.0提出一种逃逸机制，将重放的二级消息远程或本地投放到其他Master的Commitlog中。 +二级消息一般分为两个阶段，发送或者消费时会发送到一个特殊topic中，后台会有线程会扫描，最终的满足要求的消息会被重新投递到Commitlog中。我们可以让brokerId最小的Slave进行扫描，但如果扫描之后的消息重新投递到本Commitlog，那将会破坏Slave不可写的语义，造成Commitlog分叉。因此RocketMQ 5.0提出一种逃逸机制，将重放的二级消息远程或本地投放到其他Master的Commitlog中。 - 远程逃逸 From 63b9fbf75d8af04374a91688f2eed1bda157a2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Mon, 26 Aug 2024 17:44:09 +0800 Subject: [PATCH 121/265] [ISSUE #8544] Restore retry mechanism in unit test pipeline --- .github/workflows/maven.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index a49201b8a16..7d74c832be2 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -53,4 +53,14 @@ jobs: with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log - retention-days: 1 \ No newline at end of file + retention-days: 1 + + - name: Retry if failed + # if it failed , retry 2 times at most + if: failure() && fromJSON(github.run_attempt) < 3 + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Attempting to retry workflow..." + gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file From 9e6bbf7f635856acba7fd9000f8e4ba486f51b85 Mon Sep 17 00:00:00 2001 From: cnScarb Date: Tue, 27 Aug 2024 15:04:16 +0800 Subject: [PATCH 122/265] [ISSUE #8137] Support pop consumption for light message queue --- .../processor/QueryAssignmentProcessor.java | 9 +- .../QueryAssignmentProcessorTest.java | 20 ++++ .../rocketmq/example/simple/LMQProducer.java | 61 +++++++++++ .../example/simple/LMQPullConsumer.java | 76 +++++++++++++ .../example/simple/LMQPushConsumer.java | 90 +++++++++++++++ .../example/simple/LMQPushPopConsumer.java | 103 ++++++++++++++++++ 6 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java index d55f1b5b7fb..2f4cb7b15f8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java @@ -174,7 +174,14 @@ private Set doLoadBalance(final String topic, final String consume break; } case CLUSTERING: { - Set mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic); + Set mqSet; + if (MixAll.isLmq(topic)) { + mqSet = new HashSet<>(); + mqSet.add(new MessageQueue( + topic, brokerController.getBrokerConfig().getBrokerName(), (int)MixAll.LMQ_QUEUE_ID)); + } else { + mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic); + } if (null == mqSet) { if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java index e91c1a09617..67ff74897ef 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java @@ -19,8 +19,10 @@ import com.google.common.collect.ImmutableSet; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.topic.TopicRouteInfoManager; @@ -126,6 +128,24 @@ public void testSetMessageRequestMode_RetryTopic() throws Exception { assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); } + @Test + public void testDoLoadBalance() throws Exception { + Method method = queryAssignmentProcessor.getClass() + .getDeclaredMethod("doLoadBalance", String.class, String.class, String.class, MessageModel.class, + String.class, SetMessageRequestModeRequestBody.class, ChannelHandlerContext.class); + method.setAccessible(true); + + Set mqs1 = (Set) method.invoke( + queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, "127.0.0.1", MessageModel.CLUSTERING, + new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext); + Set mqs2 = (Set) method.invoke( + queryAssignmentProcessor, MixAll.LMQ_PREFIX + topic, group, "127.0.0.2", MessageModel.CLUSTERING, + new AllocateMessageQueueAveragely().getName(), new SetMessageRequestModeRequestBody(), handlerContext); + + assertThat(mqs1).hasSize(1); + assertThat(mqs2).isEmpty(); + } + @Test public void testAllocate4Pop() { testAllocate4Pop(new AllocateMessageQueueAveragely()); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java new file mode 100644 index 00000000000..81ef2e13859 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class LMQProducer { + public static final String PRODUCER_GROUP = "ProducerGroupName"; + + public static final String DEFAULT_NAMESRVADDR = "127.0.0.1:9876"; + + public static final String TOPIC = "TopicLMQParent"; + + public static final String TAG = "TagA"; + + public static final String LMQ_TOPIC_1 = MixAll.LMQ_PREFIX + "123"; + + public static final String LMQ_TOPIC_2 = MixAll.LMQ_PREFIX + "456"; + + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP); + + // Uncomment the following line while debugging, namesrvAddr should be set to your local address + producer.setNamesrvAddr(DEFAULT_NAMESRVADDR); + + producer.start(); + for (int i = 0; i < 128; i++) { + try { + Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */, + String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } catch (Exception e) { + e.printStackTrace(); + } + } + + producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java new file mode 100644 index 00000000000..7b1bdc39215 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingException; + +@SuppressWarnings("deprecation") +public class LMQPullConsumer { + public static final String BROKER_NAME = "broker-a"; + + public static final String CONSUMER_GROUP = "CID_LMQ_PULL_1"; + + public static final String TOPIC = "TopicLMQParent"; + + public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "123"; + + public static final String NAMESRV_ADDR = "127.0.0.1:9876"; + + public static void main(String[] args) throws MQClientException, RemotingException, InterruptedException { + + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(CONSUMER_GROUP); + consumer.setNamesrvAddr(NAMESRV_ADDR); + consumer.setRegisterTopics(new HashSet<>(Arrays.asList(TOPIC))); + consumer.start(); + + // use parent topic to fill up broker addr table + consumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory() + .updateTopicRouteInfoFromNameServer(TOPIC); + + final MessageQueue lmq = new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID); + long offset = consumer.minOffset(lmq); + + consumer.pullBlockIfNotFound(lmq, "*", offset, 32, new PullCallback() { + @Override + public void onSuccess(PullResult pullResult) { + List list = pullResult.getMsgFoundList(); + if (list == null || list.isEmpty()) { + return; + } + + for (MessageExt msg : list) { + System.out.printf("%s Pull New Messages: %s %n", Thread.currentThread().getName(), msg); + } + } + + @Override + public void onException(Throwable e) { + + } + }); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java new file mode 100644 index 00000000000..efe37d86816 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import com.google.common.collect.Lists; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + +public class LMQPushConsumer { + public static final String CLUSTER_NAME = "DefaultCluster"; + + public static final String BROKER_NAME = "broker-a"; + + public static final String TOPIC = "TopicLMQParent"; + + public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "123"; + + public static final String CONSUMER_GROUP = "CID_LMQ_1"; + + public static final String NAMESRV_ADDR = "127.0.0.1:9876"; + + public static final HashMap BROKER_ADDR_MAP = new HashMap() { + { + put(MixAll.MASTER_ID, "127.0.0.1:10911"); + } + }; + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP); + consumer.setNamesrvAddr(NAMESRV_ADDR); + consumer.subscribe(LMQ_TOPIC, "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + + // use parent topic to fill up broker addr table + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC); + + final TopicRouteData topicRouteData = new TopicRouteData(); + final BrokerData brokerData = new BrokerData(); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME); + brokerData.setBrokerAddrs(BROKER_ADDR_MAP); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData); + // compensate for RebalanceImpl#topicSubscribeInfoTable + consumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(LMQ_TOPIC, + new HashSet<>(Arrays.asList(new MessageQueue(LMQ_TOPIC, BROKER_NAME, (int) MixAll.LMQ_QUEUE_ID)))); + // re-balance immediately to start pulling messages + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance(); + + System.out.printf("Consumer Started.%n"); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java new file mode 100644 index 00000000000..2044057b2af --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageRequestMode; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; + +public class LMQPushPopConsumer { + public static final String CLUSTER_NAME = "DefaultCluster"; + + public static final String BROKER_NAME = "broker-a"; + + public static final String TOPIC = "TopicLMQParent"; + + public static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "456"; + + public static final String NAMESRV_ADDR = "127.0.0.1:9876"; + + public static final String CONSUMER_GROUP = "CID_LMQ_POP_1"; + + public static final HashMap BROKER_ADDR_MAP = new HashMap() { + { + put(MixAll.MASTER_ID, "127.0.0.1:10911"); + } + }; + + public static void main(String[] args) throws Exception { + switchPop(); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(CONSUMER_GROUP); + consumer.setNamesrvAddr(NAMESRV_ADDR); + consumer.subscribe(LMQ_TOPIC, "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + // use server side rebalance + consumer.setClientRebalance(false); + consumer.start(); + + // use parent topic to fill up broker addr table + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().updateTopicRouteInfoFromNameServer(TOPIC); + + final TopicRouteData topicRouteData = new TopicRouteData(); + final BrokerData brokerData = new BrokerData(); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME); + brokerData.setBrokerAddrs(BROKER_ADDR_MAP); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + // compensate LMQ topic route for MQClientInstance#findBrokerAddrByTopic + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getTopicRouteTable().put(LMQ_TOPIC, topicRouteData); + // re-balance immediately to start pulling messages + consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().doRebalance(); + + System.out.printf("Consumer Started.%n"); + } + + private static void switchPop() throws Exception { + DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); + mqAdminExt.setNamesrvAddr(NAMESRV_ADDR); + mqAdminExt.start(); + List brokerDatas = mqAdminExt.examineTopicRouteInfo(TOPIC).getBrokerDatas(); + for (BrokerData brokerData : brokerDatas) { + Set brokerAddrs = new HashSet<>(brokerData.getBrokerAddrs().values()); + for (String brokerAddr : brokerAddrs) { + mqAdminExt.setMessageRequestMode(brokerAddr, LMQ_TOPIC, CONSUMER_GROUP, MessageRequestMode.POP, 8, + 3_000); + } + } + } +} From a1ea1eb1f2b8187166b1b0e1f646e6476f27dda5 Mon Sep 17 00:00:00 2001 From: caigy Date: Tue, 27 Aug 2024 15:21:53 +0800 Subject: [PATCH 123/265] [ISSUE #8576] Support Creating or Updating Subscription Groups in Batch support creating or updating groups in batch support creating or updating groups in batch support creating or updating groups in batch --- .../processor/AdminBrokerProcessor.java | 38 ++++++ .../SubscriptionGroupManager.java | 12 ++ .../SubscriptionGroupManagerTest.java | 57 ++++++++- .../rocketmq/client/impl/MQClientAPIImpl.java | 17 +++ .../remoting/protocol/RequestCode.java | 2 + .../protocol/body/SubscriptionGroupList.java | 42 +++++++ .../tools/admin/DefaultMQAdminExt.java | 6 + .../tools/admin/DefaultMQAdminExtImpl.java | 6 + .../rocketmq/tools/admin/MQAdminExt.java | 4 + .../tools/command/MQAdminStartup.java | 2 + .../UpdateSubGroupListSubCommand.java | 119 ++++++++++++++++++ .../UpdateSubGroupListSubCommandTest.java | 45 +++++++ 12 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index c5419a62df7..3039cf5c97c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -131,6 +131,7 @@ import org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody; import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList; import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicList; @@ -282,6 +283,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return this.unlockBatchMQ(ctx, request); case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP: return this.updateAndCreateSubscriptionGroup(ctx, request); + case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST: + return this.updateAndCreateSubscriptionGroupList(ctx, request); case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG: return this.getAllSubscriptionGroup(ctx, request); case RequestCode.DELETE_SUBSCRIPTIONGROUP: @@ -1571,6 +1574,41 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c return response; } + private RemotingCommand updateAndCreateSubscriptionGroupList(ChannelHandlerContext ctx, RemotingCommand request) { + final long startTime = System.nanoTime(); + + final SubscriptionGroupList subscriptionGroupList = SubscriptionGroupList.decode(request.getBody(), SubscriptionGroupList.class); + final List groupConfigList = subscriptionGroupList.getGroupConfigList(); + + final StringBuilder builder = new StringBuilder(); + for (SubscriptionGroupConfig config : groupConfigList) { + builder.append(config.getGroupName()).append(";"); + } + final String groupNames = builder.toString(); + LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroupList: groupNames: {}, called by {}", + groupNames, + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + try { + this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfigList(groupConfigList); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } finally { + long executionTime = (System.nanoTime() - startTime) / 1000000L; + LOGGER.info("executionTime of create updateAndCreateSubscriptionGroupList: {} is {} ms", groupNames, executionTime); + InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + Attributes attributes = BrokerMetricsManager.newAttributesBuilder() + .put(LABEL_INVOCATION_STATUS, status.getName()) + .build(); + BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes); + } + + return response; + } + + private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) { String topic = topicConfig.getTopicName(); for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index 1d9614fe582..f2a7e0482b1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; @@ -138,6 +139,11 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName } public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) { + updateSubscriptionGroupConfigWithoutPersist(config); + this.persist(); + } + + private void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) { Map newAttributes = request(config); Map currentAttributes = current(config.getGroupName()); @@ -157,7 +163,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) } updateDataVersion(); + } + public void updateSubscriptionGroupConfigList(List configList) { + if (null == configList || configList.isEmpty()) { + return; + } + configList.forEach(this::updateSubscriptionGroupConfigWithoutPersist); this.persist(); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java index 3ed4ac11a40..3384d479c6e 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManagerTest.java @@ -18,11 +18,12 @@ package org.apache.rocketmq.broker.subscription; import com.google.common.collect.ImmutableMap; - import java.nio.file.Paths; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.UUID; - import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.SubscriptionGroupAttributes; @@ -39,7 +40,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + @RunWith(MockitoJUnitRunner.class) public class SubscriptionGroupManagerTest { @@ -113,4 +118,52 @@ public void updateSubscriptionGroupConfig() { private boolean notToBeExecuted() { return MixAll.isMac(); } + @Test + public void testUpdateSubscriptionGroupConfigList_NullConfigList() { + if (notToBeExecuted()) { + return; + } + + subscriptionGroupManager.updateSubscriptionGroupConfigList(null); + // Verifying that persist() is not called + verify(subscriptionGroupManager, never()).persist(); + } + + @Test + public void testUpdateSubscriptionGroupConfigList_EmptyConfigList() { + if (notToBeExecuted()) { + return; + } + + subscriptionGroupManager.updateSubscriptionGroupConfigList(Collections.emptyList()); + // Verifying that persist() is not called + verify(subscriptionGroupManager, never()).persist(); + } + + @Test + public void testUpdateSubscriptionGroupConfigList_ValidConfigList() { + if (notToBeExecuted()) { + return; + } + + final List configList = new LinkedList<>(); + final List groupNames = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + String groupName = String.format("group-%d", i); + config.setGroupName(groupName); + configList.add(config); + groupNames.add(groupName); + } + + subscriptionGroupManager.updateSubscriptionGroupConfigList(configList); + + // Verifying that persist() is called once + verify(subscriptionGroupManager, times(1)).persist(); + + groupNames.forEach(groupName -> + assertThat(subscriptionGroupManager.getSubscriptionGroupTable().get(groupName)).isNotNull()); + + } + } \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index f3d7e7c70f9..8a3d3dd0dcb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -137,6 +137,7 @@ import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody; import org.apache.rocketmq.remoting.protocol.body.SetMessageRequestModeRequestBody; +import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList; import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.remoting.protocol.body.TopicList; @@ -400,6 +401,22 @@ public void createSubscriptionGroup(final String addr, final SubscriptionGroupCo } + public void createSubscriptionGroupList(final String address, final List configs, + final long timeoutMillis) throws RemotingException, InterruptedException, MQClientException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST, null); + SubscriptionGroupList requestBody = new SubscriptionGroupList(configs); + request.setBody(requestBody.encode()); + + RemotingCommand response = this.remotingClient.invokeSync( + MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), address), request, timeoutMillis); + assert response != null; + if (response.getCode() == ResponseCode.SUCCESS) { + return; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index 3be22fc56b7..f45ff6fa484 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -148,6 +148,8 @@ public class RequestCode { public static final int GET_TOPICS_BY_CLUSTER = 224; + public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST = 225; + public static final int QUERY_TOPICS_BY_CONSUMER = 343; public static final int QUERY_SUBSCRIPTION_BY_CONSUMER = 345; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java new file mode 100644 index 00000000000..c343ce21118 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/SubscriptionGroupList.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.body; + +import java.util.List; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; + +public class SubscriptionGroupList extends RemotingSerializable { + @CFNotNull + private List groupConfigList; + + public SubscriptionGroupList() {} + + public SubscriptionGroupList(List groupConfigList) { + this.groupConfigList = groupConfigList; + } + + public List getGroupConfigList() { + return groupConfigList; + } + + public void setGroupConfigList(List groupConfigList) { + this.groupConfigList = groupConfigList; + } + +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 37dd322488f..5be6d24ff76 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -238,6 +238,12 @@ public void createAndUpdateSubscriptionGroupConfig(String addr, defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfig(addr, config); } + @Override + public void createAndUpdateSubscriptionGroupConfigList(String brokerAddr, + List configs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfigList(brokerAddr, configs); + } + @Override public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index b5a20673dab..9546235d3e8 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -318,6 +318,12 @@ public void createAndUpdateSubscriptionGroupConfig(String addr, this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroup(addr, config, timeoutMillis); } + @Override + public void createAndUpdateSubscriptionGroupConfigList(String brokerAddr, + List configs) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroupList(brokerAddr, configs, timeoutMillis); + } + @Override public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 96940c38b26..9dff3cbab95 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -118,6 +118,10 @@ void createAndUpdateSubscriptionGroupConfig(final String addr, final SubscriptionGroupConfig config) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + void createAndUpdateSubscriptionGroupConfigList(String brokerAddr, + List configs) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr, final String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index d56ed053268..43e4259c4e1 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -67,6 +67,7 @@ import org.apache.rocketmq.tools.command.consumer.GetConsumerConfigSubCommand; import org.apache.rocketmq.tools.command.consumer.SetConsumeModeSubCommand; import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand; +import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupListSubCommand; import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand; import org.apache.rocketmq.tools.command.container.AddBrokerSubCommand; import org.apache.rocketmq.tools.command.container.RemoveBrokerSubCommand; @@ -192,6 +193,7 @@ public static void initCommand() { initCommand(new UpdateTopicListSubCommand()); initCommand(new DeleteTopicSubCommand()); initCommand(new UpdateSubGroupSubCommand()); + initCommand(new UpdateSubGroupListSubCommand()); initCommand(new SetConsumeModeSubCommand()); initCommand(new DeleteSubscriptionGroupCommand()); initCommand(new UpdateBrokerConfigSubCommand()); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java new file mode 100644 index 00000000000..a36f50bd1b0 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommand.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.consumer; + +import com.alibaba.fastjson2.JSON; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +public class UpdateSubGroupListSubCommand implements SubCommand { + @Override + public String commandName() { + return "updateSubGroupList"; + } + + @Override + public String commandDesc() { + return "Update or create subscription group in batch"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + final OptionGroup optionGroup = new OptionGroup(); + Option opt = new Option("b", "brokerAddr", true, "create groups to which broker"); + optionGroup.addOption(opt); + + opt = new Option("c", "clusterName", true, "create groups to which cluster"); + optionGroup.addOption(opt); + optionGroup.setRequired(true); + options.addOptionGroup(optionGroup); + + opt = new Option("f", "filename", true, + "Path to a file with a list of org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig in json format"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + final DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + final String fileName = commandLine.getOptionValue('f').trim(); + + try { + final Path filePath = Paths.get(fileName); + if (!Files.exists(filePath)) { + System.out.printf("the file path %s does not exists%n", fileName); + return; + } + final byte[] groupConfigListBytes = Files.readAllBytes(filePath); + final List groupConfigs = JSON.parseArray(groupConfigListBytes, SubscriptionGroupConfig.class); + if (null == groupConfigs || groupConfigs.isEmpty()) { + return; + } + + if (commandLine.hasOption('b')) { + String brokerAddress = commandLine.getOptionValue('b').trim(); + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfigList(brokerAddress, groupConfigs); + + System.out.printf("submit batch of group config to %s success, please check the result later.%n", + brokerAddress); + return; + + } else if (commandLine.hasOption('c')) { + final String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String brokerAddress : masterSet) { + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfigList(brokerAddress, groupConfigs); + + System.out.printf("submit batch of subscription group config to %s success, please check the result later.%n", + brokerAddress); + } + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java new file mode 100644 index 00000000000..0c23787709b --- /dev/null +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupListSubCommandTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.consumer; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class UpdateSubGroupListSubCommandTest { + + @Test + public void testArguments() { + UpdateSubGroupListSubCommand cmd = new UpdateSubGroupListSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + + String brokerAddress = "127.0.0.1:10911"; + String inputFileName = "groups.json"; + String[] args = new String[] {"-b " + brokerAddress, "-f " + inputFileName}; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), args, + cmd.buildCommandlineOptions(options), new DefaultParser()); + + assertEquals(brokerAddress, commandLine.getOptionValue('b').trim()); + assertEquals(inputFileName, commandLine.getOptionValue('f').trim()); + } +} \ No newline at end of file From 00a05a5faa40a0c8f8deb59f0c8058e62b9bd747 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 28 Aug 2024 09:57:50 +0800 Subject: [PATCH 124/265] [ISSUE #8586] Add more test coverage for SelectMessageQueueByRandom (#8587) --- .../SelectMessageQueueByRandomTest.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java new file mode 100644 index 00000000000..9443c3f0181 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/SelectMessageQueueByRandomTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer.selector; + +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +public class SelectMessageQueueByRandomTest { + + private final SelectMessageQueueByRandom selector = new SelectMessageQueueByRandom(); + + private final String defaultBroker = "defaultBroker"; + + private final String defaultTopic = "defaultTopic"; + + @Test + public void testSelectRandomMessageQueue() { + List messageQueues = createMessageQueues(10); + Message message = new Message(defaultTopic, "tag", "key", "body".getBytes()); + MessageQueue selectedQueue = selector.select(messageQueues, message, null); + assertNotNull(selectedQueue); + assertEquals(messageQueues.size(), 10); + assertEquals(defaultTopic, selectedQueue.getTopic()); + assertEquals(defaultBroker, selectedQueue.getBrokerName()); + } + + @Test + public void testSelectEmptyMessageQueue() { + List emptyQueues = new ArrayList<>(); + Message message = new Message(defaultTopic, "tag", "key", "body".getBytes()); + assertThrows(IllegalArgumentException.class, () -> selector.select(emptyQueues, message, null)); + } + + @Test + public void testSelectSingleMessageQueue() { + List singleQueueList = createMessageQueues(1); + Message message = new Message(defaultTopic, "tag", "key", "body".getBytes()); + MessageQueue selectedQueue = selector.select(singleQueueList, message, null); + assertNotNull(selectedQueue); + assertEquals(defaultTopic, selectedQueue.getTopic()); + assertEquals(defaultBroker, selectedQueue.getBrokerName()); + assertEquals(singleQueueList.get(0).getQueueId(), selectedQueue.getQueueId()); + } + + private List createMessageQueues(final int count) { + List result = new ArrayList<>(); + for (int i = 0; i < count; i++) { + result.add(new MessageQueue(defaultTopic, defaultBroker, i)); + } + return result; + } +} From 18c30cbab653a2e5c383aace271e1972204b5291 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 29 Aug 2024 15:05:52 +0800 Subject: [PATCH 125/265] [ISSUE #8592] Not notify long polling request when pop orderly consume blocked (#8593) --- .../broker/processor/PopMessageProcessor.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 6073023722a..47ef8e4013b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -551,18 +551,24 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), true, lockKey, true); - if (isOrder && brokerController.getConsumerOrderInfoManager().checkBlock(attemptId, topic, - requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) { - future.complete(this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum); - return future; - } + // Current requests would calculate the total number of messages + // waiting to be filtered for new message arrival notifications in + // the long-polling service, need disregarding the backlog in order + // consumption scenario. If rest message num including the blocked + // queue accumulation would lead to frequent unnecessary wake-ups + // of long-polling requests, resulting unnecessary CPU usage. + // When client ack message, long-polling request would be notifications + // by AckMessageProcessor.ackOrderly() and message will not be delayed. if (isOrder) { + if (brokerController.getConsumerOrderInfoManager().checkBlock( + attemptId, topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) { + // should not add accumulation(max offset - consumer offset) here + future.complete(restNum); + return future; + } this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNum( - topic, - requestHeader.getConsumerGroup(), - queueId - ); + topic, requestHeader.getConsumerGroup(), queueId); } if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) { From b9d9b3fcae6ecdf421fba246721bd3ab984edbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Fri, 30 Aug 2024 09:20:45 +0800 Subject: [PATCH 126/265] [ISSUE #8607] Exclude loopback addresses when iterating over local network interfaces --- .../java/org/apache/rocketmq/common/utils/NetworkUtil.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java index a7a9a7c7960..2dc2a890e77 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/NetworkUtil.java @@ -129,6 +129,10 @@ public static InetAddress getLocalInetAddress() { ArrayList ipv6Result = new ArrayList<>(); List localInetAddressList = getLocalInetAddressList(); for (InetAddress inetAddress : localInetAddressList) { + // Skip loopback addresses + if (inetAddress.isLoopbackAddress()) { + continue; + } if (inetAddress instanceof Inet6Address) { ipv6Result.add(inetAddress); } else { From 1a15729ca962d76ffe044f6332ec711b1d7546bc Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Fri, 30 Aug 2024 13:43:01 +0800 Subject: [PATCH 127/265] [ISSUE #8601]When isPopShouldStop hit,unlock queueLockManager (#8602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:when isPopShouldStop hit， unlock queueLockManager * fix:when isPopShouldStop hit， unlock queueLockManager * fix: limit rate of appending commit in case of DLedger commit-log Signed-off-by: Zhanhui Li --------- Signed-off-by: Zhanhui Li Co-authored-by: Zhanhui Li --- .../rocketmq/broker/processor/PopMessageProcessor.java | 2 +- .../rocketmq/store/dledger/MessageStoreTestBase.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 47ef8e4013b..5430fdec94d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -540,6 +540,7 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, return future; } + future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) { POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId); restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; @@ -548,7 +549,6 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, } try { - future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), true, lockKey, true); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index a21806ffcf6..c4d9f0727b9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store.dledger; +import com.google.common.util.concurrent.RateLimiter; import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import java.io.File; @@ -122,7 +123,13 @@ protected DefaultMessageStore createMessageStore(String base, boolean createAbor } protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) throws UnknownHostException { + RateLimiter rateLimiter = RateLimiter.create(100); + MessageStoreConfig storeConfig = messageStore.getMessageStoreConfig(); + boolean limitAppendRate = storeConfig.isEnableDLegerCommitLog(); for (int i = 0; i < num; i++) { + if (limitAppendRate) { + rateLimiter.acquire(); + } MessageExtBrokerInner msgInner = buildMessage(); msgInner.setTopic(topic); msgInner.setQueueId(queueId); From e5e38396ba32293b3bd40a5a40ff402d42dce928 Mon Sep 17 00:00:00 2001 From: rongtong Date: Fri, 30 Aug 2024 14:15:03 +0800 Subject: [PATCH 128/265] [ISSUE #8591] Preliminary support for key commands of LMQ (#8590) * Preliminary support for key commands of LMQ * Preliminary support for key commands of LMQ * Optimize some code * Fix some bugs and UTs for lmq support * Fix UTs can not pass * Fix UTs can not pass * Add some check to prevent NPE --- .../processor/AdminBrokerProcessor.java | 2 +- .../rocketmq/client/impl/MQAdminImpl.java | 47 ++++-- .../rocketmq/client/impl/MQAdminImplTest.java | 2 +- .../example/{simple => lmq}/LMQProducer.java | 3 +- .../{simple => lmq}/LMQPullConsumer.java | 2 +- .../{simple => lmq}/LMQPushConsumer.java | 2 +- .../{simple => lmq}/LMQPushPopConsumer.java | 2 +- .../tools/admin/DefaultMQAdminExt.java | 85 ++++++++-- .../tools/admin/DefaultMQAdminExtImpl.java | 157 +++++++++++++----- .../rocketmq/tools/admin/MQAdminExt.java | 12 ++ .../consumer/ConsumerProgressSubCommand.java | 8 +- .../message/QueryMsgByIdSubCommand.java | 29 ++-- .../message/QueryMsgByKeySubCommand.java | 25 ++- .../QueryMsgByUniqueKeySubCommand.java | 28 ++-- .../offset/ResetOffsetByTimeCommand.java | 13 +- .../offset/ResetOffsetByTimeOldCommand.java | 13 +- .../offset/SkipAccumulationSubCommand.java | 7 +- .../command/topic/TopicStatusSubCommand.java | 24 ++- .../QueryMsgByUniqueKeySubCommandTest.java | 12 +- 19 files changed, 348 insertions(+), 125 deletions(-) rename example/src/main/java/org/apache/rocketmq/example/{simple => lmq}/LMQProducer.java (97%) rename example/src/main/java/org/apache/rocketmq/example/{simple => lmq}/LMQPullConsumer.java (98%) rename example/src/main/java/org/apache/rocketmq/example/{simple => lmq}/LMQPushConsumer.java (98%) rename example/src/main/java/org/apache/rocketmq/example/{simple => lmq}/LMQPushPopConsumer.java (99%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 3039cf5c97c..28bd2549145 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -2062,7 +2062,7 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId Map queueOffsetMap = new HashMap<>(); // Reset offset for all queues belonging to the specified topic - TopicConfig topicConfig = brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); if (null == topicConfig) { response.setCode(ResponseCode.TOPIC_NOT_EXIST); response.setRemark("Topic " + topic + " does not exist"); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index bcfe29bd4f6..c1e3ee33dc1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; @@ -43,6 +44,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.NetworkUtil; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -199,7 +201,7 @@ public long searchOffset(MessageQueue mq, long timestamp, BoundaryType boundaryT if (brokerAddr != null) { try { return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq, timestamp, - boundaryType, timeoutMillis); + boundaryType, timeoutMillis); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -277,13 +279,20 @@ public MessageExt viewMessage(String topic, String msgId) public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - return queryMessage(topic, key, maxNum, begin, end, false); + return queryMessage(null, topic, key, maxNum, begin, end, false); } public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - return queryMessage(topic, uniqKey, maxNum, begin, end, true); + return queryMessage(null, topic, uniqKey, maxNum, begin, end, true); + } + + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String uniqKey, int maxNum, long begin, + long end) + throws MQClientException, InterruptedException { + + return queryMessage(clusterName, topic, uniqKey, maxNum, begin, end, true); } public MessageExt queryMessageByUniqKey(String topic, @@ -311,25 +320,29 @@ public MessageExt queryMessageByUniqKey(String clusterName, String topic, } } - protected QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end, + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, boolean isUniqKey) throws MQClientException, InterruptedException { - return queryMessage(null, topic, key, maxNum, begin, end, isUniqKey); - } + boolean isLmq = MixAll.isLmq(topic); + + String routeTopic = topic; + // if topic is lmq ,then use clusterName as lmq parent topic + // Use clusterName or lmq parent topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (isLmq || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } - protected QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end, - boolean isUniqKey) throws MQClientException, - InterruptedException { - TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic); if (null == topicRouteData) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(routeTopic); + topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(routeTopic); } if (topicRouteData != null) { List brokerAddrs = new LinkedList<>(); for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { - if (clusterName != null && !clusterName.isEmpty() + if (!isLmq && clusterName != null && !clusterName.isEmpty() && !clusterName.equals(brokerData.getCluster())) { continue; } @@ -347,7 +360,11 @@ protected QueryResult queryMessage(String clusterName, String topic, String key, for (String addr : brokerAddrs) { try { QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader(); - requestHeader.setTopic(topic); + if (isLmq) { + requestHeader.setTopic(clusterName); + } else { + requestHeader.setTopic(topic); + } requestHeader.setKey(key); requestHeader.setMaxNum(maxNum); requestHeader.setBeginTimestamp(begin); @@ -436,7 +453,7 @@ public void operationFail(Throwable throwable) { String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR); for (String k : keyArray) { // both topic and key must be equal at the same time - if (Objects.equals(key, k) && Objects.equals(topic, msgTopic)) { + if (Objects.equals(key, k) && (isLmq || Objects.equals(topic, msgTopic))) { matched = true; break; } diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java index 3663df24d65..f52aba2dc00 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQAdminImplTest.java @@ -165,7 +165,7 @@ public void assertQueryMessage() throws InterruptedException, MQClientException, callback.operationSucceed(response); return null; }).when(mQClientAPIImpl).queryMessage(anyString(), any(), anyLong(), any(InvokeCallback.class), any()); - QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L, false); + QueryResult actual = mqAdminImpl.queryMessage(defaultTopic, "keys", 100, 1L, 50L); assertNotNull(actual); assertEquals(1, actual.getMessageList().size()); assertEquals(defaultTopic, actual.getMessageList().get(0).getTopic()); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java similarity index 97% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java index 81ef2e13859..5fee9480287 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -47,6 +47,7 @@ public static void main(String[] args) throws MQClientException, InterruptedExce for (int i = 0; i < 128; i++) { try { Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + msg.setKeys("Key" + i); msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */, String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); SendResult sendResult = producer.send(msg); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java similarity index 98% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java index 7b1bdc39215..931dd96b48f 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPullConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPullConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import java.util.Arrays; import java.util.HashSet; diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java similarity index 98% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java index efe37d86816..f8926a05dfd 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import com.google.common.collect.Lists; diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java similarity index 99% rename from example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java index 2044057b2af..517eb12b7d2 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LMQPushPopConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQPushPopConsumer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.example.simple; +package org.apache.rocketmq.example.lmq; import com.google.common.collect.Lists; import java.util.HashMap; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 5be6d24ff76..6ebee1d0dd1 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -153,6 +153,12 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); } + + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException, RemotingException { + return defaultMQAdminExtImpl.queryMessage(clusterName, topic, key, maxNum, begin, end); + } + @Override public void start() throws MQClientException { defaultMQAdminExtImpl.start(); @@ -196,7 +202,8 @@ public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws R } @Override - public void createAndUpdateTopicConfigList(String addr, List topicConfigList) throws InterruptedException, RemotingException, MQClientException { + public void createAndUpdateTopicConfigList(String addr, + List topicConfigList) throws InterruptedException, RemotingException, MQClientException { defaultMQAdminExtImpl.createAndUpdateTopicConfigList(addr, topicConfigList); } @@ -300,6 +307,12 @@ public ConsumeStats examineConsumeStats( return examineConsumeStats(consumerGroup, null); } + @Override + public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup, + String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineConsumeStats(clusterName, consumerGroup, topic); + } + @Override public ConsumeStats examineConsumeStats(String consumerGroup, String topic) throws RemotingException, MQClientException, @@ -459,16 +472,35 @@ public List resetOffsetByTimestampOld(String consumerGroup, Strin return defaultMQAdminExtImpl.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); } + public List resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic, long timestamp, + boolean force) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force); + } + + @Override + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce); + } + @Override public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); } + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce, boolean isC) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, isC); + } + + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, boolean isC) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - return defaultMQAdminExtImpl.resetOffsetByTimestamp(topic, group, timestamp, isForce, isC); + return defaultMQAdminExtImpl.resetOffsetByTimestamp(null, topic, group, timestamp, isForce, isC); } @Override @@ -589,10 +621,19 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c @Override public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, - final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); } + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup, + final String clientId, + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId); + } + @Override public List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, @@ -796,10 +837,10 @@ public void resetMasterFlushOffset(String brokerAddr, long masterFlushOffset) this.defaultMQAdminExtImpl.resetMasterFlushOffset(brokerAddr, masterFlushOffset); } - public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end) + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException { - - return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end); + return defaultMQAdminExtImpl.queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end); } public DefaultMQAdminExtImpl getDefaultMQAdminExtImpl() { @@ -831,13 +872,14 @@ public void updateControllerConfig(Properties properties, @Override public Pair electMaster(String controllerAddr, String clusterName, - String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { + String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { return this.defaultMQAdminExtImpl.electMaster(controllerAddr, clusterName, brokerName, brokerId); } @Override public void cleanControllerBrokerData(String controllerAddr, String clusterName, String brokerName, - String brokerControllerIdsToClean, boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException { + String brokerControllerIdsToClean, + boolean isCleanLivingBroker) throws RemotingException, InterruptedException, MQBrokerException { this.defaultMQAdminExtImpl.cleanControllerBrokerData(controllerAddr, clusterName, brokerName, brokerControllerIdsToClean, isCleanLivingBroker); } @@ -876,13 +918,15 @@ public void createUser(String brokerAddr, } @Override - public void createUser(String brokerAddr, String username, String password, String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createUser(String brokerAddr, String username, String password, + String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createUser(brokerAddr, username, password, userType); } @Override public void updateUser(String brokerAddr, String username, - String password, String userType, String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + String password, String userType, + String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateUser(brokerAddr, username, password, userType, userStatus); } @@ -912,38 +956,45 @@ public List listUser(String brokerAddr, @Override public void createAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createAcl(brokerAddr, subject, resources, actions, sourceIps, decision); } @Override - public void createAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.createAcl(brokerAddr, aclInfo); } @Override public void updateAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateAcl(brokerAddr, subject, resources, actions, sourceIps, decision); } @Override - public void updateAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.updateAcl(brokerAddr, aclInfo); } @Override - public void deleteAcl(String brokerAddr, String subject, String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void deleteAcl(String brokerAddr, String subject, + String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { defaultMQAdminExtImpl.deleteAcl(brokerAddr, subject, resource); } @Override - public AclInfo getAcl(String brokerAddr, String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public AclInfo getAcl(String brokerAddr, + String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return defaultMQAdminExtImpl.getAcl(brokerAddr, subject); } @Override - public List listAcl(String brokerAddr, String subjectFilter, String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public List listAcl(String brokerAddr, String subjectFilter, + String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return defaultMQAdminExtImpl.listAcl(brokerAddr, subjectFilter, resourceFilter); } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 9546235d3e8..dc4d35e7049 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -422,14 +422,21 @@ public KVTable fetchBrokerRuntimeStats( return this.mqClientInstance.getMQClientAPIImpl().getBrokerRuntimeInfo(brokerAddr, timeoutMillis); } + @Override + public ConsumeStats examineConsumeStats( + String consumerGroup, + String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return examineConsumeStats(null, consumerGroup, topic); + } + @Override public ConsumeStats examineConsumeStats( String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { - return examineConsumeStats(consumerGroup, null); + return examineConsumeStats(null, consumerGroup, null); } @Override - public ConsumeStats examineConsumeStats(String consumerGroup, + public ConsumeStats examineConsumeStats(String clusterName, String consumerGroup, String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { TopicRouteData topicRouteData = null; List routeTopics = new ArrayList<>(); @@ -438,6 +445,12 @@ public ConsumeStats examineConsumeStats(String consumerGroup, routeTopics.add(topic); routeTopics.add(KeyBuilder.buildPopRetryTopic(topic, consumerGroup)); } + + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) && !StringUtils.isEmpty(clusterName)) { + routeTopics.add(clusterName); + } + for (int i = 0; i < routeTopics.size(); i++) { try { topicRouteData = this.examineTopicRouteInfo(routeTopics.get(i)); @@ -467,25 +480,33 @@ public ConsumeStats examineConsumeStats(String consumerGroup, topics.add(messageQueue.getTopic()); } - ConsumeStats staticResult = new ConsumeStats(); - staticResult.setConsumeTps(result.getConsumeTps()); - // for topic, we put the physical stats, how about group? - // staticResult.getOffsetTable().putAll(result.getOffsetTable()); - - for (String currentTopic : topics) { - TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic); - if (currentRoute.getTopicQueueMappingByBroker() == null - || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { - //normal topic - for (Map.Entry entry : result.getOffsetTable().entrySet()) { - if (entry.getKey().getTopic().equals(currentTopic)) { - staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + ConsumeStats staticResult = null; + + if (StringUtils.isEmpty(clusterName)) { + + staticResult = new ConsumeStats(); + staticResult.setConsumeTps(result.getConsumeTps()); + // for topic, we put the physical stats, how about group? + // staticResult.getOffsetTable().putAll(result.getOffsetTable()); + + for (String currentTopic : topics) { + TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic); + if (currentRoute.getTopicQueueMappingByBroker() == null + || currentRoute.getTopicQueueMappingByBroker().isEmpty()) { + //normal topic + for (Map.Entry entry : result.getOffsetTable().entrySet()) { + if (entry.getKey().getTopic().equals(currentTopic)) { + staticResult.getOffsetTable().put(entry.getKey(), entry.getValue()); + } } } + Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt); + ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result); + staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable()); } - Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt); - ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result); - staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable()); + + } else { + staticResult = result; } if (staticResult.getOffsetTable().isEmpty()) { @@ -811,10 +832,16 @@ public void deleteKvConfig(String namespace, this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigValue(namespace, key, timeoutMillis); } - @Override - public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + public List resetOffsetByTimestampOld(String clusterName, String consumerGroup, String topic, + long timestamp, boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + String routeTopic = topic; + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } + TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic); List rollbackStatsList = new ArrayList<>(); Map topicRouteMap = new HashMap<>(); for (QueueData queueData : topicRouteData.getQueueDatas()) { @@ -829,6 +856,12 @@ public List resetOffsetByTimestampOld(String consumerGroup, Strin return rollbackStatsList; } + @Override + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestampOld(null, consumerGroup, topic, timestamp, force); + } + private List resetOffsetByTimestampOld(String brokerAddr, QueueData queueData, String consumerGroup, String topic, long timestamp, boolean force) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { @@ -864,7 +897,7 @@ private List resetOffsetByTimestampOld(String brokerAddr, QueueDa @Override public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); + return resetOffsetByTimestamp(null, topic, group, timestamp, isForce, false); } @Override @@ -951,9 +984,16 @@ public void run() { }); } - public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce, boolean isC) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + String routeTopic = topic; + // Use clusterName topic to get topic route for lmq or rmq_sys_wheel_timer + if (!StringUtils.isEmpty(topic) && (MixAll.isLmq(topic) || topic.equals(TopicValidator.SYSTEM_TOPIC_PREFIX + "wheel_timer")) + && !StringUtils.isEmpty(clusterName)) { + routeTopic = clusterName; + } + TopicRouteData topicRouteData = this.examineTopicRouteInfo(routeTopic); List brokerDatas = topicRouteData.getBrokerDatas(); Map allOffsetTable = new HashMap<>(); if (brokerDatas != null) { @@ -1325,7 +1365,8 @@ public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String c @Override public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, - final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { MessageExt msg = this.viewMessage(topic, msgId); if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis); @@ -1335,6 +1376,20 @@ public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumer } } + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String clusterName, final String consumerGroup, + final String clientId, + final String topic, + final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + MessageExt msg = this.queryMessage(clusterName, topic, msgId); + if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgId, timeoutMillis); + } else { + MessageClientExt msgClient = (MessageClientExt) msg; + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(NetworkUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, topic, msgClient.getOffsetMsgId(), timeoutMillis); + } + } + @Override public List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { @@ -1664,10 +1719,10 @@ public TopicConfigSerializeWrapper getUserTopicConfig(final String brokerAddr, f while (iterator.hasNext()) { TopicConfig topicConfig = iterator.next().getValue(); if (topicList.getTopicList().contains(topicConfig.getTopicName()) - || TopicValidator.isSystemTopic(topicConfig.getTopicName())) { + || TopicValidator.isSystemTopic(topicConfig.getTopicName())) { iterator.remove(); } else if (!specialTopic && StringUtils.startsWithAny(topicConfig.getTopicName(), - MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) { + MixAll.RETRY_GROUP_TOPIC_PREFIX, MixAll.DLQ_GROUP_TOPIC_PREFIX)) { iterator.remove(); } else if (!PermName.isValid(topicConfig.getPerm())) { iterator.remove(); @@ -1726,6 +1781,11 @@ public QueryResult queryMessage(String topic, String key, int maxNum, long begin return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessage(String clusterName, String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException, RemotingException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(clusterName, topic, key, maxNum, begin, end, false); + } + @Override public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { @@ -1783,10 +1843,9 @@ public long searchOffset(final String brokerAddr, final String topicName, final return this.mqClientInstance.getMQClientAPIImpl().searchOffset(brokerAddr, topicName, queueId, timestamp, timeoutMillis); } - public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, + public QueryResult queryMessageByUniqKey(String clusterName, String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { - - return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end); + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(clusterName, topic, key, maxNum, begin, end); } @Override @@ -1812,6 +1871,12 @@ public void resetOffsetByQueueId(final String brokerAddr, final String consumeGr } } + @Override + public Map resetOffsetByTimestamp(String clusterName, String topic, String group, + long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestamp(clusterName, topic, group, timestamp, isForce, false); + } + @Override public HARuntimeInfo getBrokerHAStatus( String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { @@ -1844,7 +1909,7 @@ public void resetMasterFlushOffset(String brokerAddr, @Override public Pair electMaster(String controllerAddr, String clusterName, - String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { + String brokerName, Long brokerId) throws RemotingException, InterruptedException, MQBrokerException { return this.mqClientInstance.getMQClientAPIImpl().electMaster(controllerAddr, clusterName, brokerName, brokerId); } @@ -1930,20 +1995,23 @@ public void createUser(String brokerAddr, } @Override - public void createUser(String brokerAddr, String username, String password, String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createUser(String brokerAddr, String username, String password, + String userType) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { UserInfo userInfo = UserInfo.of(username, password, userType); this.createUser(brokerAddr, userInfo); } @Override public void updateUser(String brokerAddr, String username, - String password, String userType, String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + String password, String userType, + String userStatus) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { UserInfo userInfo = UserInfo.of(username, password, userType, userStatus); this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis); } @Override - public void updateUser(String brokerAddr, UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateUser(String brokerAddr, + UserInfo userInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().updateUser(brokerAddr, userInfo, timeoutMillis); } @@ -1967,40 +2035,47 @@ public List listUser(String brokerAddr, @Override public void createAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision); this.createAcl(brokerAddr, aclInfo); } @Override - public void createAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void createAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().createAcl(brokerAddr, aclInfo, timeoutMillis); } @Override public void updateAcl(String brokerAddr, String subject, List resources, List actions, - List sourceIps, String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + List sourceIps, + String decision) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { AclInfo aclInfo = AclInfo.of(subject, resources, actions, sourceIps, decision); this.updateAcl(brokerAddr, aclInfo); } @Override - public void updateAcl(String brokerAddr, AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void updateAcl(String brokerAddr, + AclInfo aclInfo) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().updateAcl(brokerAddr, aclInfo, timeoutMillis); } @Override - public void deleteAcl(String brokerAddr, String subject, String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public void deleteAcl(String brokerAddr, String subject, + String resource) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { this.mqClientInstance.getMQClientAPIImpl().deleteAcl(brokerAddr, subject, resource, timeoutMillis); } @Override - public AclInfo getAcl(String brokerAddr, String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public AclInfo getAcl(String brokerAddr, + String subject) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return this.mqClientInstance.getMQClientAPIImpl().getAcl(brokerAddr, subject, timeoutMillis); } @Override - public List listAcl(String brokerAddr, String subjectFilter, String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { + public List listAcl(String brokerAddr, String subjectFilter, + String resourceFilter) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQBrokerException, InterruptedException { return this.mqClientInstance.getMQClientAPIImpl().listAcl(brokerAddr, subjectFilter, resourceFilter, timeoutMillis); } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 9dff3cbab95..ff78f22c704 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -152,6 +152,10 @@ ConsumeStats examineConsumeStats(final String consumerGroup, final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + ConsumeStats examineConsumeStats(final String clusterName, final String consumerGroup, + final String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException; + ConsumeStats examineConsumeStats(final String brokerAddr, final String consumerGroup, final String topicName, final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException; @@ -232,6 +236,9 @@ List resetOffsetByTimestampOld(String consumerGroup, String topic Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + Map resetOffsetByTimestamp(String clusterName, String topic, String group, long timestamp, boolean isForce) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; @@ -293,6 +300,11 @@ ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String topic, String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + ConsumeMessageDirectlyResult consumeMessageDirectly(String clusterName, String consumerGroup, + String clientId, + String topic, + String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + List messageTrackDetail( MessageExt msg) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java index c489cad6849..b638dcf61f3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -72,6 +72,10 @@ public Options buildCommandlineOptions(Options options) { optionShowClientIP.setRequired(false); options.addOption(optionShowClientIP); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -109,6 +113,8 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t boolean showClientIP = commandLine.hasOption('s') && "true".equalsIgnoreCase(commandLine.getOptionValue('s')); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; + if (commandLine.hasOption('g')) { String consumerGroup = commandLine.getOptionValue('g').trim(); String topicName = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : null; @@ -116,7 +122,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (topicName == null) { consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); } else { - consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup, topicName); + consumeStats = defaultMQAdminExt.examineConsumeStats(clusterName, consumerGroup, topicName); } List mqList = new LinkedList<>(consumeStats.getOffsetTable().keySet()); Collections.sort(mqList); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java index 5245ca089ff..e83029eed31 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -44,9 +44,10 @@ import org.apache.rocketmq.tools.command.SubCommandException; public class QueryMsgByIdSubCommand implements SubCommand { - public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, final Charset msgBodyCharset) throws MQClientException, + public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic, + final String msgId, final Charset msgBodyCharset) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, IOException { - MessageExt msg = admin.viewMessage(topic, msgId); + MessageExt msg = admin.queryMessage(clusterName, topic, msgId); printMsg(admin, msg, msgBodyCharset); } @@ -55,7 +56,8 @@ public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg) printMsg(admin, msg, null); } - public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg, final Charset msgBodyCharset) throws IOException { + public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg, + final Charset msgBodyCharset) throws IOException { if (msg == null) { System.out.printf("%nMessage not found!"); return; @@ -219,6 +221,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -244,13 +250,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t final String msgIds = commandLine.getOptionValue('i').trim(); final String[] msgIdArr = StringUtils.split(msgIds, ","); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); final String clientId = commandLine.getOptionValue('d').trim(); for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - pushMsg(defaultMQAdminExt, consumerGroup, clientId, topic, msgId.trim()); + pushMsg(defaultMQAdminExt, clusterName, consumerGroup, clientId, topic, msgId.trim()); } } } else if (commandLine.hasOption('s')) { @@ -258,7 +265,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (resend) { for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - sendMsg(defaultMQAdminExt, defaultMQProducer, topic, msgId.trim()); + sendMsg(defaultMQAdminExt, clusterName, defaultMQProducer, topic, msgId.trim()); } } } @@ -269,7 +276,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } for (String msgId : msgIdArr) { if (StringUtils.isNotBlank(msgId)) { - queryById(defaultMQAdminExt, topic, msgId.trim(), msgBodyCharset); + queryById(defaultMQAdminExt, clusterName, topic, msgId.trim(), msgBodyCharset); } } @@ -282,13 +289,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } } - private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String consumerGroup, final String clientId, + private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName, + final String consumerGroup, final String clientId, final String topic, final String msgId) { try { ConsumerRunningInfo consumerRunningInfo = defaultMQAdminExt.getConsumerRunningInfo(consumerGroup, clientId, false, false); if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) { ConsumeMessageDirectlyResult result = - defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + defaultMQAdminExt.consumeMessageDirectly(clusterName, consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { System.out.printf("this %s client is not push consumer ,not support direct push \n", clientId); @@ -298,10 +306,11 @@ private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String con } } - private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final DefaultMQProducer defaultMQProducer, + private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final String clusterName, + final DefaultMQProducer defaultMQProducer, final String topic, final String msgId) { try { - MessageExt msg = defaultMQAdminExt.viewMessage(topic, msgId); + MessageExt msg = defaultMQAdminExt.queryMessage(clusterName, topic, msgId); if (msg != null) { // resend msg by id System.out.printf("prepare resend msg. originalMsgId=%s", msgId); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java index 64627fd19fa..02961c3bb50 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java @@ -23,6 +23,8 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; + import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; @@ -41,7 +43,7 @@ public String commandDesc() { @Override public Options buildCommandlineOptions(Options options) { - Option opt = new Option("t", "topic", true, "topic name"); + Option opt = new Option("t", "topic", true, "Topic name"); opt.setRequired(true); options.addOption(opt); @@ -57,7 +59,11 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); - opt = new Option("c", "maxNum", true, "The maximum number of messages returned by the query, default:64"); + opt = new Option("m", "maxNum", true, "The maximum number of messages returned by the query, default:64"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); opt.setRequired(false); options.addOption(opt); @@ -77,16 +83,20 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t long beginTimestamp = 0; long endTimestamp = Long.MAX_VALUE; int maxNum = 64; + String clusterName = null; if (commandLine.hasOption("b")) { beginTimestamp = Long.parseLong(commandLine.getOptionValue("b").trim()); } if (commandLine.hasOption("e")) { endTimestamp = Long.parseLong(commandLine.getOptionValue("e").trim()); } + if (commandLine.hasOption("m")) { + maxNum = Integer.parseInt(commandLine.getOptionValue("m").trim()); + } if (commandLine.hasOption("c")) { - maxNum = Integer.parseInt(commandLine.getOptionValue("c").trim()); + clusterName = commandLine.getOptionValue("c").trim(); } - this.queryByKey(defaultMQAdminExt, topic, key, maxNum, beginTimestamp, endTimestamp); + this.queryByKey(defaultMQAdminExt, clusterName, topic, key, maxNum, beginTimestamp, endTimestamp); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } finally { @@ -94,12 +104,13 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } } - private void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key, int maxNum, long begin, + private void queryByKey(final DefaultMQAdminExt admin, final String cluster, final String topic, final String key, int maxNum, long begin, long end) - throws MQClientException, InterruptedException { + throws MQClientException, InterruptedException, RemotingException { admin.start(); - QueryResult queryResult = admin.queryMessage(topic, key, maxNum, begin, end); + QueryResult queryResult = admin.queryMessage(cluster, topic, key, maxNum, begin, end); + System.out.printf("%-50s %4s %40s%n", "#Message ID", "#QID", diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index b71cee90160..5295d91cc30 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -25,13 +25,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.rocketmq.client.QueryResult; -import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; -import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; @@ -51,19 +49,18 @@ private DefaultMQAdminExt createMQAdminExt(RPCHook rpcHook) throws SubCommandExc defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); try { defaultMQAdminExt.start(); - } - catch (Exception e) { + } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } return defaultMQAdminExt; } } - public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, - final boolean showAll) throws MQClientException, - RemotingException, MQBrokerException, InterruptedException, IOException { + public static void queryById(final DefaultMQAdminExt admin, final String clusterName, final String topic, + final String msgId, + final boolean showAll) throws MQClientException, InterruptedException, IOException { - QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE); + QueryResult queryResult = admin.queryMessageByUniqKey(clusterName, topic, msgId, 32, 0, Long.MAX_VALUE); assert queryResult != null; List list = queryResult.getMessageList(); if (list == null || list.size() == 0) { @@ -94,7 +91,7 @@ private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, i System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())); System.out.printf(intFormat, "System Flag:", msg.getSysFlag()); System.out.printf(strFormat, "Properties:", - msg.getProperties() != null ? msg.getProperties().toString() : ""); + msg.getProperties() != null ? msg.getProperties().toString() : ""); System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath); try { @@ -166,6 +163,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -173,10 +174,11 @@ public Options buildCommandlineOptions(Options options) { public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { try { - defaultMQAdminExt = createMQAdminExt(rpcHook); + defaultMQAdminExt = createMQAdminExt(rpcHook); final String msgId = commandLine.getOptionValue('i').trim(); final String topic = commandLine.getOptionValue('t').trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; final boolean showAll = commandLine.hasOption('a'); if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); @@ -189,14 +191,14 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } if (consumerRunningInfo != null && ConsumerRunningInfo.isPushType(consumerRunningInfo)) { ConsumeMessageDirectlyResult result = - defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { - System.out.printf("get consumer info failed or this %s client is not push consumer ,not support direct push \n", clientId); + System.out.printf("get consumer info failed or this %s client is not push consumer, not support direct push \n", clientId); } } else { - queryById(defaultMQAdminExt, topic, msgId, showAll); + queryById(defaultMQAdminExt, clusterName, topic, msgId, showAll); } } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java index 993fa501875..84a301bd60c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java @@ -77,6 +77,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -88,6 +92,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t String group = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); String timeStampStr = commandLine.getOptionValue("s").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; long timestamp = "now".equals(timeStampStr) ? System.currentTimeMillis() : 0; try { @@ -129,7 +134,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t if (brokerAddr != null && queueId >= 0) { System.out.printf("start reset consumer offset by specified, " + "group[%s], topic[%s], queueId[%s], broker[%s], timestamp(string)[%s], timestamp(long)[%s]%n", - group, topic, queueId, brokerAddr, timeStampStr, timestamp); + group, topic, queueId, brokerAddr, timeStampStr, timestamp); long resetOffset = null != offset ? offset : defaultMQAdminExt.searchOffset(brokerAddr, topic, queueId, timestamp, 3000); @@ -143,11 +148,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t Map offsetTable; try { - offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force, isC); + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force, isC); } catch (MQClientException e) { - // if consumer not online, use old command to reset reset + // if consumer not online, use old command to reset if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { - ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, group, topic, timestamp, force, timeStampStr); + ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, clusterName, group, topic, timestamp, force, timeStampStr); return; } throw e; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java index 7984bb8c39f..c179c5c8051 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java @@ -34,12 +34,13 @@ public class ResetOffsetByTimeOldCommand implements SubCommand { - public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String consumerGroup, String topic, + public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String clusterName, String consumerGroup, + String topic, long timestamp, boolean force, String timeStampStr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { List rollbackStatsList = - defaultMQAdminExt.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + defaultMQAdminExt.resetOffsetByTimestampOld(clusterName, consumerGroup, topic, timestamp, force); System.out.printf("reset consumer offset by specified " + "consumerGroup[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]%n", @@ -93,6 +94,11 @@ public Options buildCommandlineOptions(Options options) { opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); opt.setRequired(false); options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -104,6 +110,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t String consumerGroup = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); String timeStampStr = commandLine.getOptionValue("s").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; long timestamp = 0; try { timestamp = Long.parseLong(timeStampStr); @@ -123,7 +130,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t force = Boolean.parseBoolean(commandLine.getOptionValue("f").trim()); } defaultMQAdminExt.start(); - resetOffset(defaultMQAdminExt, consumerGroup, topic, timestamp, force, timeStampStr); + resetOffset(defaultMQAdminExt, clusterName, consumerGroup, topic, timestamp, force, timeStampStr); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java index b22491a5918..8f2ac2e1e14 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java @@ -57,6 +57,10 @@ public Options buildCommandlineOptions(Options options) { opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); opt.setRequired(false); options.addOption(opt); + + opt = new Option("c", "cluster", true, "Cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -68,6 +72,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t try { String group = commandLine.getOptionValue("g").trim(); String topic = commandLine.getOptionValue("t").trim(); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; boolean force = true; if (commandLine.hasOption('f')) { force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); @@ -76,7 +81,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t defaultMQAdminExt.start(); Map offsetTable; try { - offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force); + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(clusterName, topic, group, timestamp, force); } catch (MQClientException e) { if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { List rollbackStatsList = defaultMQAdminExt.resetOffsetByTimestampOld(group, topic, timestamp, force); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java index a1619ecedfd..47ca761d1f6 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java @@ -27,6 +27,8 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; @@ -48,6 +50,10 @@ public Options buildCommandlineOptions(Options options) { Option opt = new Option("t", "topic", true, "topic name"); opt.setRequired(true); options.addOption(opt); + + opt = new Option("c", "cluster", true, "cluster name or lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -58,10 +64,26 @@ public void execute(final CommandLine commandLine, final Options options, defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + TopicStatsTable topicStatsTable = new TopicStatsTable(); defaultMQAdminExt.start(); String topic = commandLine.getOptionValue('t').trim(); - TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + + if (commandLine.hasOption('c')) { + String cluster = commandLine.getOptionValue('c').trim(); + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(cluster); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = defaultMQAdminExt.examineTopicStats(addr, topic); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + } else { + topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + } List mqList = new LinkedList<>(); mqList.addAll(topicStatsTable.getOffsetTable().keySet()); diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java index fc5405e7472..b24bd22db8f 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommandTest.java @@ -127,7 +127,7 @@ public void before() throws NoSuchFieldException, IllegalAccessException, Interr when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString())).thenReturn(retMsgExt); QueryResult queryResult = new QueryResult(0, Lists.newArrayList(retMsgExt)); - when(defaultMQAdminExtImpl.queryMessageByUniqKey(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult); + when(mQAdminImpl.queryMessageByUniqKey(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(queryResult); TopicRouteData topicRouteData = new TopicRouteData(); List brokerDataList = new ArrayList<>(); @@ -194,7 +194,7 @@ public void testExecuteConsumeActively() throws SubCommandException, Interrupted Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i msgId"}; + String[] args = new String[] {"-t myTopicTest", "-i msgId", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -218,7 +218,7 @@ public void testExecuteConsumePassively() throws SubCommandException, Interrupte Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066"}; + String[] args = new String[] {"-t myTopicTest", "-i 7F000001000004D20000000000000066", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -230,7 +230,7 @@ public void testExecuteWithConsumerGroupAndClientId() throws SubCommandException Options options = ServerUtil.buildCommandlineOptions(new Options()); - String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"}; + String[] args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"}; CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); @@ -241,13 +241,13 @@ public void testExecute() throws SubCommandException { System.setProperty("rocketmq.namesrv.addr", "127.0.0.1:9876"); - String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000"}; + String[] args = new String[]{"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-c DefaultCluster"}; Options options = ServerUtil.buildCommandlineOptions(new Options()); CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); - args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId"}; + args = new String[] {"-t myTopicTest", "-i 0A3A54F7BF7D18B4AAC28A3FA2CF0000", "-g producerGroupName", "-d clientId", "-c DefaultCluster"}; commandLine = ServerUtil.parseCmdLine("mqadmin ", args, cmd.buildCommandlineOptions(options), new DefaultParser()); cmd.execute(commandLine, options, null); From 71ec1eda49277a47815159cd3d118854c8dcb3c4 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 30 Aug 2024 14:20:20 +0800 Subject: [PATCH 129/265] [ISSUE #8483] Optimize unnecessary broker reverse notification (notifyConsumerIdsChanged) in broadcast mode (#8484) * [ISSUE #8483] Optimize unnecessary broker reverse notification (notifyConsumerIdsChanged) in broadcast mode * Update * Update test * Update test --- .../broker/client/ConsumerManager.java | 15 ++- .../broker/client/ConsumerManagerTest.java | 93 ++++++++++--------- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java index 9f838b51544..b1057e2a8d4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java @@ -145,8 +145,9 @@ public boolean doChannelCloseEvent(final String remoteAddr, final Channel channe callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey()); } } - - callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); + if (!isBroadcastMode(info.getMessageModel())) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); + } } } return removed; @@ -196,7 +197,7 @@ public boolean registerConsumer(final String group, final ClientChannelInfo clie } if (r1 || r2) { - if (isNotifyConsumerIdsChangedEnable) { + if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } @@ -219,7 +220,7 @@ public boolean registerConsumerWithoutSub(final String group, final ClientChanne consumerGroupInfo = prev != null ? prev : tmp; } boolean updateChannelRst = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere); - if (updateChannelRst && isNotifyConsumerIdsChangedEnable) { + if (updateChannelRst && isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } if (null != this.brokerStatsManager) { @@ -244,7 +245,7 @@ public void unregisterConsumer(final String group, final ClientChannelInfo clien callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group); } } - if (isNotifyConsumerIdsChangedEnable) { + if (isNotifyConsumerIdsChangedEnable && !isBroadcastMode(consumerGroupInfo.getMessageModel())) { callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } @@ -334,4 +335,8 @@ protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String gr } } } + + private boolean isBroadcastMode(final MessageModel messageModel) { + return MessageModel.BROADCASTING.equals(messageModel); + } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java index 8c909824348..a23ad20037c 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java @@ -18,10 +18,7 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; -import java.util.HashSet; -import java.util.Set; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -30,13 +27,25 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -49,19 +58,10 @@ public class ConsumerManagerTest { private ConsumerManager consumerManager; - private DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener; - @Mock private BrokerController brokerController; - @Mock - private ConsumerFilterManager consumerFilterManager; - - private BrokerConfig brokerConfig = new BrokerConfig(); - - private Broker2Client broker2Client; - - private BrokerStatsManager brokerStatsManager; + private final BrokerConfig brokerConfig = new BrokerConfig(); private static final String GROUP = "DEFAULT_GROUP"; @@ -74,40 +74,38 @@ public class ConsumerManagerTest { @Before public void before() { clientChannelInfo = new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, VERSION); - defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController); - brokerStatsManager = new BrokerStatsManager(brokerConfig); - consumerManager = new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig); - broker2Client = new Broker2Client(brokerController); + DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController); + BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig); + consumerManager = spy(new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig)); + ConsumerFilterManager consumerFilterManager = mock(ConsumerFilterManager.class); when(brokerController.getConsumerFilterManager()).thenReturn(consumerFilterManager); - when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); - when(brokerController.getBroker2Client()).thenReturn(broker2Client); } @Test public void compensateBasicConsumerInfoTest() { ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNull(); + assertThat(consumerGroupInfo).isNull(); consumerManager.compensateBasicConsumerInfo(GROUP, ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING); consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNotNull(); - Assertions.assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY); - Assertions.assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING); + assertThat(consumerGroupInfo).isNotNull(); + assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY); + assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING); } @Test public void compensateSubscribeDataTest() { ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNull(); + assertThat(consumerGroupInfo).isNull(); consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true); - Assertions.assertThat(consumerGroupInfo).isNotNull(); - Assertions.assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1); + assertThat(consumerGroupInfo).isNotNull(); + assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1); SubscriptionData subscriptionData = consumerGroupInfo.getSubscriptionTable().get(TOPIC); - Assertions.assertThat(subscriptionData).isNotNull(); - Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); - Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); + assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); + assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); } @Test @@ -118,7 +116,8 @@ public void registerConsumerTest() { subList.add(subscriptionData); consumerManager.registerConsumer(GROUP, clientChannelInfo, ConsumeType.CONSUME_PASSIVELY, MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true); - Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull(); + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull(); } @Test @@ -128,63 +127,65 @@ public void unregisterConsumerTest() { // unregister consumerManager.unregisterConsumer(GROUP, clientChannelInfo, true); - Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull(); + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull(); } @Test public void findChannelTest() { register(); final ClientChannelInfo consumerManagerChannel = consumerManager.findChannel(GROUP, CLIENT_ID); - Assertions.assertThat(consumerManagerChannel).isNotNull(); + assertThat(consumerManagerChannel).isNotNull(); } @Test public void findSubscriptionDataTest() { register(); final SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC); - Assertions.assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData).isNotNull(); } @Test public void findSubscriptionDataCountTest() { register(); final int count = consumerManager.findSubscriptionDataCount(GROUP); - assert count > 0; + assertTrue(count > 0); } @Test public void findSubscriptionTest() { SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true); - Assertions.assertThat(subscriptionData).isNull(); + assertThat(subscriptionData).isNull(); consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true); - Assertions.assertThat(subscriptionData).isNotNull(); - Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); - Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); + assertThat(subscriptionData).isNotNull(); + assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC); + assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL); subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, false); - Assertions.assertThat(subscriptionData).isNull(); + assertThat(subscriptionData).isNull(); } @Test public void scanNotActiveChannelTest() { clientChannelInfo.setLastUpdateTimestamp(System.currentTimeMillis() - brokerConfig.getChannelExpiredTimeout() * 2); consumerManager.scanNotActiveChannel(); - Assertions.assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0); + assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0); } @Test public void queryTopicConsumeByWhoTest() { register(); final HashSet consumeGroup = consumerManager.queryTopicConsumeByWho(TOPIC); - assert consumeGroup.size() > 0; + assertFalse(consumeGroup.isEmpty()); } @Test public void doChannelCloseEventTest() { consumerManager.doChannelCloseEvent("127.0.0.1", channel); - assert consumerManager.getConsumerTable().size() == 0; + verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any()); + assertEquals(0, consumerManager.getConsumerTable().size()); } private void register() { @@ -203,8 +204,8 @@ public void removeExpireConsumerGroupInfo() { consumerManager.compensateSubscribeData(GROUP, TOPIC, subscriptionData); consumerManager.compensateSubscribeData(GROUP, TOPIC + "_1", new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL)); consumerManager.removeExpireConsumerGroupInfo(); - Assertions.assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull(); - Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull(); - Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull(); + assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull(); + assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull(); + assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull(); } } From 720c87e85dd95be8e56166a7f00c652ce0f1d458 Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Fri, 30 Aug 2024 14:53:51 +0800 Subject: [PATCH 130/265] [ISSUE #8584] fix missing brokerName in sendMessageBack request (#8585) * fix missing brokerName in sendMessageBack request * fix --- .../apache/rocketmq/client/consumer/DefaultMQPullConsumer.java | 2 +- .../apache/rocketmq/client/consumer/DefaultMQPushConsumer.java | 2 +- .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 2 +- .../client/impl/consumer/DefaultMQPushConsumerImplTest.java | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 089fd39b3e9..7c9a65ecdbf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -262,7 +262,7 @@ public void setRegisterTopics(Set registerTopics) { public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { msg.setTopic(withNamespace(msg.getTopic())); - this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, null); + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName()); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 94785c69708..5df5cc8fa1a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -688,7 +688,7 @@ public void setSubscription(Map subscription) { public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { msg.setTopic(withNamespace(msg.getTopic())); - this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, (String) null); + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, msg.getBrokerName()); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 0fef8666cb5..c92cadf5057 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -752,7 +752,7 @@ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerN public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - sendMessageBack(msg, delayLevel, null, mq); + sendMessageBack(msg, delayLevel, msg.getBrokerName(), mq); } diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java index 68563c02562..2bc9c5a18db 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImplTest.java @@ -651,10 +651,11 @@ public void testQueryMessageByUniqKey() throws InterruptedException, MQClientExc @Test public void testSendMessageBack() throws InterruptedException, MQClientException, MQBrokerException, RemotingException { + when(mQClientFactory.findBrokerAddressInPublish(anyString())).thenReturn(defaultBrokerAddr); defaultMQPushConsumerImpl.sendMessageBack(createMessageExt(), 1, createMessageQueue()); verify(mqClientAPIImpl).consumerSendMessageBack( eq(defaultBrokerAddr), - any(), + eq(defaultBroker), any(MessageExt.class), any(), eq(1), From 4c51706b28c192c647e186a65aec8d690f9b30f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Wed, 4 Sep 2024 11:14:34 +0800 Subject: [PATCH 131/265] [ISSUE #8623] Temporarily skip flaky unit tests on macOS (#8633) * Skip flaky tests on macOS * Trigger ci * Remove branch trigger --- .../rocketmq/store/dledger/DLedgerCommitlogTest.java | 7 +++++++ .../rocketmq/store/dledger/DLedgerMultiPathTest.java | 1 + .../apache/rocketmq/store/dledger/MixCommitlogTest.java | 3 +++ 3 files changed, 11 insertions(+) diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 1e4bbf21bd2..386cb1f6787 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.store.StoreCheckpoint; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.Assume; import org.apache.rocketmq.common.MixAll; @@ -51,6 +52,12 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { + @BeforeClass + public static void beforeClass() { + // Temporarily skip those tests on the macOS as they are flaky + Assume.assumeFalse(MixAll.isMac()); + } + @Test public void testTruncateCQ() throws Exception { String base = createBaseDir(); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java index 5eb83207322..9de4e4820ed 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerMultiPathTest.java @@ -39,6 +39,7 @@ public class DLedgerMultiPathTest extends MessageStoreTestBase { @Test public void multiDirsStorageTest() throws Exception { + Assume.assumeFalse(MixAll.isMac()); Assume.assumeFalse(MixAll.isWindows()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index db7b594a73b..1bfc6f72eaa 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -34,6 +34,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { @Test public void testFallBehindCQ() throws Exception { Assume.assumeFalse(MixAll.isWindows()); + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); @@ -75,6 +76,7 @@ public void testFallBehindCQ() throws Exception { @Test public void testPutAndGet() throws Exception { + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); @@ -138,6 +140,7 @@ public void testPutAndGet() throws Exception { @Test public void testDeleteExpiredFiles() throws Exception { + Assume.assumeFalse(MixAll.isMac()); String base = createBaseDir(); String topic = UUID.randomUUID().toString(); String peers = String.format("n0-localhost:%d", nextPort()); From 632943ea843eaf43144a008dc28fc013b80aba90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Wed, 4 Sep 2024 14:55:52 +0800 Subject: [PATCH 132/265] [ISSUE #8596] Remove redundant mvn test and log check steps from CI workflow --- .github/workflows/bazel.yml | 1 + .github/workflows/maven.yaml | 10 +--------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 268a06a79fd..5aa4f460c7c 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -27,6 +27,7 @@ jobs: - name: Retry if failed # if it failed , retry 2 times at most if: failure() && fromJSON(github.run_attempt) < 3 + continue-on-error: true env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 7d74c832be2..f17c20b1ab8 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -30,9 +30,6 @@ jobs: - name: Build with Maven run: mvn -B package --file pom.xml - - name: Run tests with increased memory and debug info - run: mvn test -X -Dparallel=none -DargLine="-Xmx1024m -XX:MaxPermSize=256m" - - name: Upload Auth JVM crash logs if: failure() uses: actions/upload-artifact@v4 @@ -41,12 +38,6 @@ jobs: path: /Users/runner/work/rocketmq/rocketmq/auth/hs_err_pid*.log retention-days: 1 - - name: Check for broker JVM crash logs - if: failure() - run: | - echo "Checking for JVM crash logs..." - ls -al /Users/runner/work/rocketmq/rocketmq/broker/ - - name: Upload broker JVM crash logs if: failure() uses: actions/upload-artifact@v4 @@ -58,6 +49,7 @@ jobs: - name: Retry if failed # if it failed , retry 2 times at most if: failure() && fromJSON(github.run_attempt) < 3 + continue-on-error: true env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 5d43e7d111ab2461c94fc62c1021cf7aee9d0219 Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:14:45 +0800 Subject: [PATCH 133/265] [ISSUE #8609] Add the BrokerConfig updateNameServerAddrPeriod (#8626) --- .../apache/rocketmq/broker/BrokerController.java | 2 +- .../org/apache/rocketmq/common/BrokerConfig.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 145a9522306..22ac7fedf1c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -747,7 +747,7 @@ public void run() { LOG.error("Failed to update nameServer address list", e); } } - }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + }, 1000 * 10, this.brokerConfig.getUpdateNameServerAddrPeriod(), TimeUnit.MILLISECONDS); } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 10bf7f76e86..26afe593a25 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -185,6 +185,11 @@ public class BrokerConfig extends BrokerIdentity { */ private int registerNameServerPeriod = 1000 * 30; + /** + * This configurable item defines interval of update name server address. Default: 120 * 1000 milliseconds + */ + private int updateNameServerAddrPeriod = 1000 * 120; + /** * the interval to send heartbeat to name server for liveness detection. */ @@ -1837,4 +1842,12 @@ public boolean isSkipWhenCKRePutReachMaxTimes() { public void setSkipWhenCKRePutReachMaxTimes(boolean skipWhenCKRePutReachMaxTimes) { this.skipWhenCKRePutReachMaxTimes = skipWhenCKRePutReachMaxTimes; } + + public int getUpdateNameServerAddrPeriod() { + return updateNameServerAddrPeriod; + } + + public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) { + this.updateNameServerAddrPeriod = updateNameServerAddrPeriod; + } } From 3368c06e6bbd04ece05703857db420e90116cbbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Thu, 5 Sep 2024 10:13:02 +0800 Subject: [PATCH 134/265] [ISSUE #8643] Add an integration testing pipeline to current CI workflow (#8644) * Add a it pipeline to workflows * Trigger * Rename it job * Remove branch trigger * Remove the step of uploading report --- .github/workflows/integration-test.yml | 38 +++++++++++++++++++ pom.xml | 15 ++++++++ .../client/consumer/pop/NotificationIT.java | 2 + .../rocketmq/test/grpc/v2/ClusterGrpcIT.java | 2 + .../rocketmq/test/grpc/v2/LocalGrpcIT.java | 2 + 5 files changed, 59 insertions(+) create mode 100644 .github/workflows/integration-test.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 00000000000..33b0a01ad4f --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,38 @@ +name: Run Integration Tests +on: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: [master, develop] + +jobs: + it-test: + name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + jdk: [8] + steps: + - name: Cache Maven Repos + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK ${{ matrix.jdk }} + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.jdk }} + distribution: "adopt" + cache: "maven" + + - name: Run integration tests with Maven + run: mvn clean verify -Pit-test -Pskip-unit-tests + + diff --git a/pom.xml b/pom.xml index 41fc3db0c40..8449bd6fb88 100644 --- a/pom.xml +++ b/pom.xml @@ -526,6 +526,21 @@ https://builds.apache.org/analysis + + skip-unit-tests + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + true + true + + + + + diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java index b5d79d6c0ae..3a6ad060020 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/NotificationIT.java @@ -30,6 +30,7 @@ import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; +import org.junit.Ignore; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -53,6 +54,7 @@ public void setUp() { } @Test + @Ignore public void testNotification() throws Exception { long pollTime = 500; CompletableFuture future1 = client.notification(brokerAddr, topic, group, messageQueue.getQueueId(), pollTime, System.currentTimeMillis(), 5000); diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java index 33c3aa2fb89..77f5f362125 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java @@ -30,6 +30,7 @@ import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.Ignore; import org.junit.runners.MethodSorters; import static org.awaitility.Awaitility.await; @@ -87,6 +88,7 @@ public void testTransactionCheckThenCommit() { } @Test + @Ignore public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java index 7f837adebe5..515c3f121dd 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; +import org.junit.Ignore; import org.junit.runners.MethodSorters; @FixMethodOrder(value = MethodSorters.NAME_ASCENDING) @@ -75,6 +76,7 @@ public void testTransactionCheckThenCommit() { } @Test + @Ignore public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } From 0b0f8ec946e58b2e032328b0f06960962afcca03 Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:16:43 +0800 Subject: [PATCH 135/265] [ISSUE #8599] Fix send fail without retry when get GO_AWAY twice (#8603) --- .../org/apache/rocketmq/client/producer/DefaultMQProducer.java | 3 ++- .../apache/rocketmq/client/producer/DefaultMQProducerTest.java | 2 +- .../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 3ecd5987c35..b47c01f6764 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -79,7 +79,8 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { ResponseCode.SYSTEM_BUSY, ResponseCode.NO_PERMISSION, ResponseCode.NO_BUYER_ID, - ResponseCode.NOT_IN_CURRENT_UNIT + ResponseCode.NOT_IN_CURRENT_UNIT, + ResponseCode.GO_AWAY )); /** diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 96086c7a255..be277f69bcf 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -769,7 +769,7 @@ public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAcces @Test public void assertGetRetryResponseCodes() { assertNotNull(producer.getRetryResponseCodes()); - assertEquals(7, producer.getRetryResponseCodes().size()); + assertEquals(8, producer.getRetryResponseCodes().size()); } @Test diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 6f61e75e01a..9f3136195b3 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -278,6 +278,7 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin "please go away"); response.setOpaque(opaque); writeResponse(ctx.channel(), cmd, response); + log.info("proxy is shutting down, write response GO_AWAY. channel={}, requestCode={}, opaque={}", ctx.channel(), cmd.getCode(), opaque); return; } } From d855ebc96f1dff85aa26bfc9253dca413eba586d Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 6 Sep 2024 09:38:51 +0800 Subject: [PATCH 136/265] [ISSUE #8640] Add more test coverage for Broker2Client (#8641) * [ISSUE #8640] Add more test coverage for Broker2Client * Update --- .../broker/client/net/Broker2ClientTest.java | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java new file mode 100644 index 00000000000..865e7b608ea --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client.net; + +import io.netty.channel.Channel; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; +import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.store.MessageStore; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class Broker2ClientTest { + + @Mock + private BrokerController brokerController; + + @Mock + private RemotingServer remotingServer; + + @Mock + private ConsumerManager consumerManager; + + @Mock + private TopicConfigManager topicConfigManager; + + @Mock + private ConsumerOffsetManager consumerOffsetManager; + + @Mock + private Channel channel; + + @Mock + private ConsumerGroupInfo consumerGroupInfo; + + private Broker2Client broker2Client; + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultGroup = "defaultGroup"; + + private final long timestamp = System.currentTimeMillis(); + + private final boolean isForce = true; + + @Before + public void init() { + broker2Client = new Broker2Client(brokerController); + when(brokerController.getRemotingServer()).thenReturn(remotingServer); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + when(brokerController.getConsumerManager()).thenReturn(consumerManager); + when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); + when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class)); + when(brokerController.getMessageStore()).thenReturn(mock(MessageStore.class)); + when(consumerManager.getConsumerGroupInfo(any())).thenReturn(consumerGroupInfo); + } + + @Test + public void testCheckProducerTransactionState() throws Exception { + CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader(); + broker2Client.checkProducerTransactionState("group", channel, requestHeader, createMessageExt()); + verify(remotingServer).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L)); + } + + @Test + public void testCheckProducerTransactionStateException() throws Exception { + CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader(); + MessageExt messageExt = createMessageExt(); + doThrow(new RuntimeException("Test Exception")) + .when(remotingServer) + .invokeOneway(any(Channel.class), + any(RemotingCommand.class), + anyLong()); + broker2Client.checkProducerTransactionState("group", channel, requestHeader, messageExt); + verify(brokerController.getRemotingServer()).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L)); + } + + @Test + public void testResetOffsetNoTopicConfig() { + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testResetOffsetNoConsumerGroupInfo() { + TopicConfig topicConfig = mock(TopicConfig.class); + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); + when(topicConfig.getWriteQueueNums()).thenReturn(1); + when(consumerOffsetManager.queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode()); + } + + @Test + public void testResetOffset() { + TopicConfig topicConfig = mock(TopicConfig.class); + when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); + when(topicConfig.getWriteQueueNums()).thenReturn(1); + when(brokerController.getConsumerOffsetManager().queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L); + BrokerConfig brokerConfig = mock(BrokerConfig.class); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + when(brokerConfig.getBrokerName()).thenReturn(defaultBroker); + ConsumerGroupInfo consumerGroupInfo = mock(ConsumerGroupInfo.class); + when(consumerManager.getConsumerGroupInfo(defaultGroup)).thenReturn(consumerGroupInfo); + RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); + assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode()); + } + + @Test + public void testGetConsumeStatusNoConsumerOnline() { + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(new ConcurrentHashMap<>()); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testGetConsumeStatusClientDoesNotSupportFeature() { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, "defaultClientId", null, MQVersion.Version.V3_0_6.ordinal()); + ConcurrentMap channelInfoTable = new ConcurrentHashMap<>(); + channelInfoTable.put(channel, clientChannelInfo); + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + + @Test + public void testGetConsumeStatus() throws Exception { + ConcurrentMap channelInfoTable = new ConcurrentHashMap<>(); + ClientChannelInfo clientChannelInfo = mock(ClientChannelInfo.class); + when(clientChannelInfo.getVersion()).thenReturn(MQVersion.CURRENT_VERSION); + channelInfoTable.put(channel, clientChannelInfo); + when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable); + RemotingCommand responseMock = mock(RemotingCommand.class); + when(responseMock.getCode()).thenReturn(ResponseCode.SUCCESS); + when(responseMock.getBody()).thenReturn("{\"consumerTable\":{}}".getBytes(StandardCharsets.UTF_8)); + when(remotingServer.invokeSync(any(Channel.class), any(RemotingCommand.class), anyLong())).thenReturn(responseMock); + RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, ""); + assertEquals(ResponseCode.SUCCESS, response.getCode()); + GetConsumerStatusBody body = RemotingSerializable.decode(response.getBody(), GetConsumerStatusBody.class); + assertEquals(1, body.getConsumerTable().size()); + } + + private MessageExt createMessageExt() { + MessageExt result = new MessageExt(); + result.setBody("body".getBytes(StandardCharsets.UTF_8)); + result.setTopic(defaultTopic); + result.setBrokerName(defaultBroker); + result.putUserProperty("key", "value"); + result.getProperties().put(MessageConst.PROPERTY_PRODUCER_GROUP, defaultGroup); + result.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "TX1"); + result.setKeys("keys"); + SocketAddress bornHost = new InetSocketAddress("127.0.0.1", 12911); + SocketAddress storeHost = new InetSocketAddress("127.0.0.1", 10911); + result.setStoreHost(storeHost); + result.setBornHost(bornHost); + return result; + } +} From 97b318f2da53e9dfb8dd12d5c3672347f7791a12 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 6 Sep 2024 09:39:47 +0800 Subject: [PATCH 137/265] [ISSUE #8649] Fix Generate coverage report ci error (#8650) * [ISSUE #8649] Fix Generate coverage report ci error * Update * Update * Update --- .../ReplicasManagerRegisterTest.java | 193 +++++++++--------- 1 file changed, 96 insertions(+), 97 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java index d01a6f76f5e..39ec0d8d94f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/controller/ReplicasManagerRegisterTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.broker.controller; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.slave.SlaveSynchronize; @@ -36,29 +37,31 @@ import org.apache.rocketmq.store.ha.autoswitch.BrokerMetadata; import org.apache.rocketmq.store.ha.autoswitch.TempBrokerMetadata; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.io.File; import java.time.Duration; -import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.UUID; import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) -@PrepareForTest(ReplicasManager.class) +@RunWith(MockitoJUnitRunner.class) public class ReplicasManagerRegisterTest { public static final String STORE_BASE_PATH = System.getProperty("java.io.tmpdir") + File.separator + "ReplicasManagerRegisterTest"; @@ -74,7 +77,14 @@ public class ReplicasManagerRegisterTest { public static final String CONTROLLER_ADDR = "127.0.0.1:8888"; public static final BrokerConfig BROKER_CONFIG; - private final HashSet syncStateSet = new HashSet<>(Arrays.asList(1L)); + + private final HashSet syncStateSet = new HashSet<>(Collections.singletonList(1L)); + + @Mock + private BrokerMetadata brokerMetadata; + + @Mock + private TempBrokerMetadata tempBrokerMetadata; static { BROKER_CONFIG = new BrokerConfig(); @@ -133,18 +143,19 @@ public void testBrokerRegisterSuccess() throws Exception { when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); + when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())) + .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); ReplicasManager replicasManager0 = new ReplicasManager(mockedBrokerController); replicasManager0.start(); await().atMost(Duration.ofMillis(1000)).until(() -> replicasManager0.getState() == ReplicasManager.State.RUNNING ); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); - Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); + assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); replicasManager0.shutdown(); } @@ -160,18 +171,18 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws await().atMost(Duration.ofMillis(1000)).until(() -> replicasManager0.getState() == ReplicasManager.State.RUNNING ); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); - Assert.assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager0.getRegisterState()); + assertEquals(1L, replicasManager0.getBrokerControllerId().longValue()); checkMetadataFile(replicasManager0.getBrokerMetadata(), 1L); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); - Assert.assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager0.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManager0.getTempBrokerMetadata().fileExists()); replicasManager0.shutdown(); // change broker name in broker config mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME + "1"); ReplicasManager replicasManagerRestart = new ReplicasManager(mockedBrokerController); replicasManagerRestart.start(); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); mockedBrokerController.getBrokerConfig().setBrokerName(BROKER_NAME); replicasManagerRestart.shutdown(); @@ -179,7 +190,7 @@ public void testBrokerRegisterSuccessAndRestartWithChangedBrokerConfig() throws mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME + "1"); replicasManagerRestart = new ReplicasManager(mockedBrokerController); replicasManagerRestart.start(); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManagerRestart.getRegisterState()); mockedBrokerController.getBrokerConfig().setBrokerClusterName(CLUSTER_NAME); replicasManagerRestart.shutdown(); } @@ -190,32 +201,29 @@ public void testRegisterFailedAtGetNextBrokerId() throws Exception { when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenThrow(new RuntimeException()); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState()); - Assert.assertFalse(replicasManager.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(replicasManager.getBrokerControllerId()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.INITIAL, replicasManager.getRegisterState()); + assertFalse(replicasManager.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManager.getBrokerMetadata().fileExists()); + assertNull(replicasManager.getBrokerControllerId()); replicasManager.shutdown(); } @Test public void testRegisterFailedAtCreateTempFile() throws Exception { - ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); + ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); - when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); - when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); - ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager); - PowerMockito.doReturn(false).when(spyReplicasManager, "createTempMetadataFile", anyLong()); + FieldUtils.writeDeclaredField(spyReplicasManager, "tempBrokerMetadata", tempBrokerMetadata, true); + doThrow(new RuntimeException("Test exception")).when(tempBrokerMetadata).updateAndPersist(any(), any(), anyLong(), any()); spyReplicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState()); - Assert.assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(spyReplicasManager.getBrokerControllerId()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.INITIAL, spyReplicasManager.getRegisterState()); + assertFalse(spyReplicasManager.getTempBrokerMetadata().fileExists()); + assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); + assertNull(spyReplicasManager.getBrokerControllerId()); spyReplicasManager.shutdown(); } @@ -224,61 +232,57 @@ public void testRegisterFailedAtApplyBrokerIdFailed() throws Exception { ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenThrow(new RuntimeException()); - when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); - Assert.assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertNotEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); + assertNotEquals(ReplicasManager.RegisterState.REGISTERED, replicasManager.getRegisterState()); replicasManager.shutdown(); - - Assert.assertFalse(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(replicasManager.getBrokerControllerId()); + + assertFalse(replicasManager.getBrokerMetadata().fileExists()); + assertNull(replicasManager.getBrokerControllerId()); } @Test public void testRegisterFailedAtCreateMetadataFileAndDeleteTemp() throws Exception { - ReplicasManager replicasManager = new ReplicasManager(mockedBrokerController); + ReplicasManager spyReplicasManager = new ReplicasManager(mockedBrokerController); when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 1L)); when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), anyLong(), any(), any())).thenReturn(new ApplyBrokerIdResponseHeader()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); - - ReplicasManager spyReplicasManager = PowerMockito.spy(replicasManager); - PowerMockito.doReturn(false).when(spyReplicasManager, "createMetadataFileAndDeleteTemp"); + + FieldUtils.writeDeclaredField(spyReplicasManager, "brokerMetadata", brokerMetadata, true); + doThrow(new RuntimeException("Test exception")).when(brokerMetadata).updateAndPersist(any(), any(), anyLong()); spyReplicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, spyReplicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.CREATE_TEMP_METADATA_FILE_DONE, spyReplicasManager.getRegisterState()); TempBrokerMetadata tempBrokerMetadata = spyReplicasManager.getTempBrokerMetadata(); - Assert.assertTrue(tempBrokerMetadata.fileExists()); - Assert.assertTrue(tempBrokerMetadata.isLoaded()); - Assert.assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); - Assert.assertNull(spyReplicasManager.getBrokerControllerId()); + assertTrue(tempBrokerMetadata.fileExists()); + assertTrue(tempBrokerMetadata.isLoaded()); + assertFalse(spyReplicasManager.getBrokerMetadata().fileExists()); + assertNull(spyReplicasManager.getBrokerControllerId()); spyReplicasManager.shutdown(); // restart, we expect that this replicasManager still keep the tempMetadata and still try to finish its registering ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController); - // because apply brokerId: 1 has succeeded, so now next broker id is 2 - when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L)); replicasManagerNew.start(); - - Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); + + assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); // tempMetadata has been cleared - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); + assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); + assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); replicasManagerNew.shutdown(); } @@ -291,62 +295,57 @@ public void testRegisterFailedAtRegisterSuccess() throws Exception { when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); replicasManager.start(); - - Assert.assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); + + assertEquals(ReplicasManager.State.FIRST_TIME_SYNC_CONTROLLER_METADATA_DONE, replicasManager.getState()); + assertEquals(ReplicasManager.RegisterState.CREATE_METADATA_FILE_DONE, replicasManager.getRegisterState()); TempBrokerMetadata tempBrokerMetadata = replicasManager.getTempBrokerMetadata(); // temp metadata has been cleared - Assert.assertFalse(tempBrokerMetadata.fileExists()); - Assert.assertFalse(tempBrokerMetadata.isLoaded()); + assertFalse(tempBrokerMetadata.fileExists()); + assertFalse(tempBrokerMetadata.isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManager.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManager.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManager.getBrokerControllerId().longValue()); + assertTrue(replicasManager.getBrokerMetadata().fileExists()); + assertTrue(replicasManager.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManager.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManager.getBrokerControllerId().longValue()); replicasManager.shutdown(); Mockito.reset(mockedBrokerOuterAPI); - when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())).thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); + when(mockedBrokerOuterAPI.brokerElect(any(), any(), any(), anyLong())) + .thenReturn(new Pair<>(new ElectMasterResponseHeader(1L, "127.0.0.1:13131", 1, 1), syncStateSet)); when(mockedBrokerOuterAPI.getControllerMetaData(any())).thenReturn( new GetMetaDataResponseHeader("default-group", "dledger-a", CONTROLLER_ADDR, true, CONTROLLER_ADDR)); when(mockedBrokerOuterAPI.checkAddressReachable(any())).thenReturn(true); // restart, we expect that this replicasManager still keep the metadata and still try to finish its registering ReplicasManager replicasManagerNew = new ReplicasManager(mockedBrokerController); - // because apply brokerId: 1 has succeeded, so now next broker id is 2 - when(mockedBrokerOuterAPI.getNextBrokerId(any(), any(), any())).thenReturn(new GetNextBrokerIdResponseHeader(CLUSTER_NAME, BROKER_NAME, 2L)); - // because apply brokerId: 1 has succeeded, so next request which try to apply brokerId: 1 will be failed - when(mockedBrokerOuterAPI.applyBrokerId(any(), any(), eq(1L), any(), any())).thenThrow(new RuntimeException()); when(mockedBrokerOuterAPI.registerBrokerToController(any(), any(), anyLong(), any(), any())).thenReturn(new Pair<>(new RegisterBrokerToControllerResponseHeader(), syncStateSet)); replicasManagerNew.start(); - - Assert.assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); - Assert.assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); + + assertEquals(ReplicasManager.State.RUNNING, replicasManagerNew.getState()); + assertEquals(ReplicasManager.RegisterState.REGISTERED, replicasManagerNew.getRegisterState()); // tempMetadata has been cleared - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); - Assert.assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().fileExists()); + assertFalse(replicasManagerNew.getTempBrokerMetadata().isLoaded()); // metadata has been persisted - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); - Assert.assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); - Assert.assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); + assertTrue(replicasManagerNew.getBrokerMetadata().fileExists()); + assertTrue(replicasManagerNew.getBrokerMetadata().isLoaded()); + assertEquals(1L, replicasManagerNew.getBrokerMetadata().getBrokerId().longValue()); + assertEquals(1L, replicasManagerNew.getBrokerControllerId().longValue()); replicasManagerNew.shutdown(); } private void checkMetadataFile(BrokerMetadata brokerMetadata0 ,Long brokerId) throws Exception { - Assert.assertEquals(brokerId, brokerMetadata0.getBrokerId()); - Assert.assertTrue(brokerMetadata0.fileExists()); + assertEquals(brokerId, brokerMetadata0.getBrokerId()); + assertTrue(brokerMetadata0.fileExists()); BrokerMetadata brokerMetadata = new BrokerMetadata(brokerMetadata0.getFilePath()); brokerMetadata.readFromFile(); - Assert.assertEquals(brokerMetadata0, brokerMetadata); + assertEquals(brokerMetadata0, brokerMetadata); } @After public void clear() { UtilAll.deleteFile(new File(STORE_BASE_PATH)); } - - } From 5496807572f3803d2c6fc3d0f1671dc5f149a2f9 Mon Sep 17 00:00:00 2001 From: rongtong Date: Fri, 6 Sep 2024 16:15:08 +0800 Subject: [PATCH 138/265] [ISSUE #8647] Fix the issue where lmq cannot update consumer offset (#8648) * Fix the issue where lmq cannot update consumer offset * Fix the issue where lmq cannot update consumer offset --- .../broker/subscription/LmqSubscriptionGroupManager.java | 9 +++++++++ .../subscription/RocksDBLmqSubscriptionGroupManager.java | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java index 018083811e8..69e59fd8e7f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java @@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) } super.updateSubscriptionGroupConfig(config); } + + @Override + public boolean containsSubscriptionGroup(String group) { + if (MixAll.isLmq(group)) { + return true; + } else { + return super.containsSubscriptionGroup(group); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java index 8c05d0bd98f..e4de25756bf 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java @@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) } super.updateSubscriptionGroupConfig(config); } + + @Override + public boolean containsSubscriptionGroup(String group) { + if (MixAll.isLmq(group)) { + return true; + } else { + return super.containsSubscriptionGroup(group); + } + } } From 9b53c06f07ffd484676baf222d3d1df31e91bd4b Mon Sep 17 00:00:00 2001 From: Ji Juntao Date: Sat, 7 Sep 2024 19:58:42 +0800 Subject: [PATCH 139/265] [ISSUE #8657] Make retry topic pop probability configurable --- .../broker/processor/PopMessageProcessor.java | 2 +- .../java/org/apache/rocketmq/common/BrokerConfig.java | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 5430fdec94d..2d76c5a3caa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -373,7 +373,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC // considered the same type because they share the same retry flag in previous fields. // Therefore, needRetryV1 is designed as a subset of needRetry, and within a single request, // only one type of retry topic is able to call popMsgFromQueue. - boolean needRetry = randomQ % 5 == 0; + boolean needRetry = randomQ < brokerConfig.getPopFromRetryProbability(); boolean needRetryV1 = false; if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) { needRetryV1 = randomQ % 2 == 0; diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 26afe593a25..2123e9b339d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -231,7 +231,7 @@ public class BrokerConfig extends BrokerIdentity { // read message from pop retry topic v1, for the compatibility, will be removed in the future version private boolean retrieveMessageFromPopRetryTopicV1 = true; private boolean enableRetryTopicV2 = false; - + private int popFromRetryProbability = 20; private boolean realTimeNotifyConsumerChange = true; private boolean litePullMessageEnable = true; @@ -563,6 +563,15 @@ public void setEnablePopLog(boolean enablePopLog) { this.enablePopLog = enablePopLog; } + public int getPopFromRetryProbability() { + return popFromRetryProbability; + } + + public void setPopFromRetryProbability(int popFromRetryProbability) { + this.popFromRetryProbability = popFromRetryProbability; + } + + public boolean isTraceOn() { return traceOn; } From e7eda2c95763a79921f39627e6a0fcbf3b8442c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Mon, 9 Sep 2024 16:32:19 +0800 Subject: [PATCH 140/265] [ISSUE #8668] Improve CI pipeline reliability with better log viewing and test fixes (#8667) * Add sleep 3s after consumer shutdown to ensure queue rebanlance * Add a step to upload test report * Unified code style * Unified mockito runner configuration to suppress UnnecessaryStubbingException --- .github/workflows/integration-test.yml | 9 ++++++++- .../client/consumer/DefaultLitePullConsumerTest.java | 5 +---- .../client/consumer/DefaultMQPushConsumerTest.java | 4 +++- .../consumer/balance/NormalMsgDynamicBalanceIT.java | 2 ++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 33b0a01ad4f..8179f362879 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -35,4 +35,11 @@ jobs: - name: Run integration tests with Maven run: mvn clean verify -Pit-test -Pskip-unit-tests - + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() + with: + report_paths: 'test/target/failsafe-reports/TEST-*.xml' + annotate_only: true + include_passed: true + detailed_summary: true diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index 65237bc8f76..592c247057b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -63,8 +63,6 @@ import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.quality.Strictness; -import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; @@ -81,8 +79,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) -@MockitoSettings(strictness = Strictness.LENIENT) +@RunWith(MockitoJUnitRunner.Silent.class) public class DefaultLitePullConsumerTest { @Spy private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index a10fd74b34f..834be5cf16f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -209,7 +209,9 @@ public PullResult answer(InvocationOnMock mock) throws Throwable { @AfterClass public static void terminate() { - pushConsumer.shutdown(); + if (pushConsumer != null) { + pushConsumer.shutdown(); + } } @Test diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java index 684b718ae5d..7408a092c4b 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgDynamicBalanceIT.java @@ -96,6 +96,8 @@ public void test3ConsumerAndCrashOne() { MQWait.waitConsumeAll(CONSUME_TIME, producer.getAllMsgBody(), consumer1.getListener(), consumer2.getListener(), consumer3.getListener()); consumer3.shutdown(); + TestUtils.waitForSeconds(WAIT_TIME); + producer.clearMsg(); consumer1.clearMsg(); consumer2.clearMsg(); From 587e9767e93ed7734dac4cf2b2d64d61529128f1 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Mon, 9 Sep 2024 16:57:12 +0800 Subject: [PATCH 141/265] [ISSUE #8653] Fix index service upload last file when broker shutdown and fetcher check in tiered storage (#8654) * [ISSUE #8653] Fix index service upload last file when broker shutdown and fetcher check in tiered storage * [ISSUE #8653] Fix index service upload last file when broker shutdown and fetcher check in tiered storage * remove unused import --- .../tieredstore/TieredMessageStore.java | 16 ++++-- .../tieredstore/index/IndexStoreService.java | 56 ++++++++++--------- .../index/IndexStoreServiceTest.java | 4 +- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java index 7b63e16696e..0e3ede871c3 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java @@ -180,9 +180,15 @@ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int } // determine whether tiered storage path conditions are met - if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK) - && !next.checkInStoreByConsumeOffset(topic, queueId, offset)) { - return true; + if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_DISK)) { + // return true to read from tiered storage if the CommitLog is empty + if (next != null && next.getCommitLog() != null && + next.getCommitLog().getMinOffset() < 0L) { + return true; + } + if (!next.checkInStoreByConsumeOffset(topic, queueId, offset)) { + return true; + } } if (storageLevel.check(MessageStoreConfig.TieredStorageLevel.NOT_IN_MEM) @@ -208,10 +214,10 @@ public CompletableFuture getMessageAsync(String group, String } if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { - log.trace("GetMessageAsync from current store, " + + log.trace("GetMessageAsync from remote store, " + "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums); } else { - log.trace("GetMessageAsync from remote store, " + + log.trace("GetMessageAsync from next store, " + "topic: {}, queue: {}, offset: {}, maxCount: {}", topic, queueId, offset, maxMsgNums); return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java index 020b9f3b068..0db5dc5c4c5 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java @@ -42,8 +42,6 @@ import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.common.AppendResult; -import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode; -import org.apache.rocketmq.tieredstore.exception.TieredStoreException; import org.apache.rocketmq.tieredstore.file.FlatAppendFile; import org.apache.rocketmq.tieredstore.file.FlatFileFactory; import org.apache.rocketmq.tieredstore.provider.FileSegment; @@ -271,23 +269,23 @@ public CompletableFuture> queryAsync( public void forceUpload() { try { readWriteLock.writeLock().lock(); - if (this.currentWriteFile == null) { - log.warn("IndexStoreService no need force upload current write file"); - return; - } - // note: current file has been shutdown before - IndexStoreFile lastFile = new IndexStoreFile(storeConfig, currentWriteFile.getTimestamp()); - if (this.doCompactThenUploadFile(lastFile)) { - this.setCompactTimestamp(lastFile.getTimestamp()); - } else { - throw new TieredStoreException( - TieredStoreErrorCode.UNKNOWN, "IndexStoreService force compact current file error"); + while (true) { + Map.Entry entry = + this.timeStoreTable.higherEntry(this.compactTimestamp.get()); + if (entry == null) { + break; + } + if (this.doCompactThenUploadFile(entry.getValue())) { + this.setCompactTimestamp(entry.getValue().getTimestamp()); + // The total number of files will not too much, prevent io too fast. + TimeUnit.MILLISECONDS.sleep(50); + } } } catch (Exception e) { log.error("IndexStoreService force upload error", e); throw new RuntimeException(e); } finally { - readWriteLock.writeLock().lock(); + readWriteLock.writeLock().unlock(); } } @@ -393,19 +391,13 @@ protected IndexFile getNextSealedFile() { @Override public void shutdown() { super.shutdown(); - readWriteLock.writeLock().lock(); - try { - for (Map.Entry entry : timeStoreTable.entrySet()) { - entry.getValue().shutdown(); - } - if (!autoCreateNewFile) { - this.forceUpload(); + // Wait index service upload then clear time store table + while (!this.timeStoreTable.isEmpty()) { + try { + TimeUnit.MILLISECONDS.sleep(50); + } catch (InterruptedException e) { + throw new RuntimeException(e); } - this.timeStoreTable.clear(); - } catch (Exception e) { - log.error("IndexStoreService shutdown error", e); - } finally { - readWriteLock.writeLock().unlock(); } } @@ -424,6 +416,18 @@ public void run() { } this.waitForRunning(TimeUnit.SECONDS.toMillis(10)); } + readWriteLock.writeLock().lock(); + try { + if (autoCreateNewFile) { + this.forceUpload(); + } + this.timeStoreTable.forEach((timestamp, file) -> file.shutdown()); + this.timeStoreTable.clear(); + } catch (Exception e) { + log.error("IndexStoreService shutdown error", e); + } finally { + readWriteLock.writeLock().unlock(); + } log.info(this.getServiceName() + " service shutdown"); } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java index fb563f7c6c2..83b407e73ba 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java @@ -120,7 +120,7 @@ public void doConvertOldFormatTest() throws IOException { indexService = new IndexStoreService(fileAllocator, filePath); indexService.start(); ConcurrentSkipListMap timeStoreTable = indexService.getTimeStoreTable(); - Assert.assertEquals(1, timeStoreTable.size()); + Assert.assertEquals(2, timeStoreTable.size()); Assert.assertEquals(Long.valueOf(timestamp), timeStoreTable.firstKey()); mappedFile.destroy(10 * 1000); } @@ -232,7 +232,7 @@ public void restartServiceTest() throws InterruptedException { indexService = new IndexStoreService(fileAllocator, filePath); indexService.start(); Assert.assertEquals(timestamp, indexService.getTimeStoreTable().firstKey().longValue()); - Assert.assertEquals(2, indexService.getTimeStoreTable().size()); + Assert.assertEquals(4, indexService.getTimeStoreTable().size()); Assert.assertEquals(IndexFile.IndexStatusEnum.UPLOAD, indexService.getTimeStoreTable().firstEntry().getValue().getFileStatus()); } From adf110fbaa9459160c155520f0e5db4a81284d42 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Mon, 9 Sep 2024 16:59:44 +0800 Subject: [PATCH 142/265] [ISSUE #8660] Should use read only getConsumeQueue instead of findConsumeQueue in read only func (#8659) --- .../rocketmq/store/DefaultMessageStore.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index f159c31a7be..8f564d5bc14 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -985,7 +985,7 @@ public long getMaxOffsetInQueue(String topic, int queueId) { @Override public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { if (committed) { - ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId); if (logic != null) { return logic.getMaxOffsetInQueue(); } @@ -1021,7 +1021,7 @@ public void setTimerMessageStore(TimerMessageStore timerMessageStore) { @Override public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); if (cqUnit != null) { @@ -1157,7 +1157,7 @@ public boolean getLastMappedFile(long startOffset) { @Override public long getEarliestMessageTime(String topic, int queueId) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { Pair pair = logicQueue.getEarliestUnitAndStoreTime(); if (pair != null && pair.getObject2() != null) { @@ -1189,7 +1189,7 @@ public long getEarliestMessageTime() { @Override public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); if (pair != null && pair.getObject2() != null) { @@ -1207,12 +1207,12 @@ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int q @Override public long getMessageTotalInQueue(String topic, int queueId) { - ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + ConsumeQueueInterface logicQueue = this.getConsumeQueue(topic, queueId); if (logicQueue != null) { return logicQueue.getMessageTotalInQueue(); } - return -1; + return 0; } @Override @@ -1496,7 +1496,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, final long maxOffsetPy = this.commitLog.getMaxOffset(); - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit cqUnit = consumeQueue.get(consumeOffset); @@ -1512,7 +1512,7 @@ public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, @Override public boolean checkInMemByConsumeOffset(final String topic, final int queueId, long consumeOffset, int batchSize) { - ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + ConsumeQueueInterface consumeQueue = getConsumeQueue(topic, queueId); if (consumeQueue != null) { CqUnit firstCQItem = consumeQueue.get(consumeOffset); if (firstCQItem == null) { From f43d9d1072050a72fc8aa68f37d3a90f663b8faa Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 9 Sep 2024 19:41:16 +0800 Subject: [PATCH 143/265] [ISSUE #8665] Add more test coverage for RebalanceLockManager (#8666) --- .../rebalance/RebalanceLockManagerTest.java | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java new file mode 100644 index 00000000000..e231d61b6a7 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManagerTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client.rebalance; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.common.message.MessageQueue; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RebalanceLockManagerTest { + + @Mock + private RebalanceLockManager.LockEntry lockEntry; + + private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager(); + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + private final String defaultGroup = "defaultGroup"; + + private final String defaultClientId = "defaultClientId"; + + @Test + public void testIsLockAllExpiredGroupNotExist() { + assertTrue(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testIsLockAllExpiredGroupExist() throws IllegalAccessException { + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + when(lockEntry.isExpired()).thenReturn(false); + assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testIsLockAllExpiredGroupExistSomeExpired() throws IllegalAccessException { + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + when(lockEntry.isExpired()).thenReturn(true).thenReturn(false); + assertFalse(rebalanceLockManager.isLockAllExpired(defaultGroup)); + } + + @Test + public void testTryLockNotLocked() { + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockSameClient() throws IllegalAccessException { + when(lockEntry.isLocked(defaultClientId)).thenReturn(true); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockDifferentClient() throws Exception { + when(lockEntry.isLocked(defaultClientId)).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertFalse(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockButExpired() throws IllegalAccessException { + when(lockEntry.isExpired()).thenReturn(true); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + assertTrue(rebalanceLockManager.tryLock(defaultGroup, createDefaultMessageQueue(), defaultClientId)); + } + + @Test + public void testTryLockBatchAllLocked() { + Set mqs = createMessageQueue(2); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId); + assertEquals(mqs, actual); + } + + @Test + public void testTryLockBatchNoneLocked() throws IllegalAccessException { + when(lockEntry.isLocked(defaultClientId)).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, createMessageQueue(2), defaultClientId); + assertTrue(actual.isEmpty()); + } + + @Test + public void testTryLockBatchSomeLocked() throws IllegalAccessException { + Set mqs = new HashSet<>(); + MessageQueue mq1 = new MessageQueue(defaultTopic, defaultBroker, 0); + MessageQueue mq2 = new MessageQueue(defaultTopic, defaultBroker, 1); + mqs.add(mq1); + mqs.add(mq2); + when(lockEntry.isLocked(defaultClientId)).thenReturn(true).thenReturn(false); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", createMQLockTable(), true); + Set actual = rebalanceLockManager.tryLockBatch(defaultGroup, mqs, defaultClientId); + Set expected = new HashSet<>(); + expected.add(mq2); + assertEquals(expected, actual); + } + + @Test + public void testUnlockBatch() throws IllegalAccessException { + when(lockEntry.getClientId()).thenReturn(defaultClientId); + ConcurrentMap> mqLockTable = createMQLockTable(); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true); + rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId); + assertEquals(1, mqLockTable.get(defaultGroup).values().size()); + } + + @Test + public void testUnlockBatchByOtherClient() throws IllegalAccessException { + when(lockEntry.getClientId()).thenReturn("otherClientId"); + ConcurrentMap> mqLockTable = createMQLockTable(); + FieldUtils.writeDeclaredField(rebalanceLockManager, "mqLockTable", mqLockTable, true); + rebalanceLockManager.unlockBatch(defaultGroup, createMessageQueue(1), defaultClientId); + assertEquals(2, mqLockTable.get(defaultGroup).values().size()); + } + + private MessageQueue createDefaultMessageQueue() { + return createMessageQueue(1).iterator().next(); + } + + private Set createMessageQueue(final int count) { + Set result = new HashSet<>(); + for (int i = 0; i < count; i++) { + result.add(new MessageQueue(defaultTopic, defaultBroker, i)); + } + return result; + } + + private ConcurrentMap> createMQLockTable() { + MessageQueue messageQueue1 = new MessageQueue(defaultTopic, defaultBroker, 0); + MessageQueue messageQueue2 = new MessageQueue(defaultTopic, defaultBroker, 1); + ConcurrentHashMap lockEntryMap = new ConcurrentHashMap<>(); + lockEntryMap.put(messageQueue1, lockEntry); + lockEntryMap.put(messageQueue2, lockEntry); + ConcurrentMap> result = new ConcurrentHashMap<>(); + result.put(defaultGroup, lockEntryMap); + return result; + } +} From ba8148f746a9ba6244de365485d7245cb52b90c4 Mon Sep 17 00:00:00 2001 From: Dongyuan Pan Date: Wed, 11 Sep 2024 17:06:40 +0800 Subject: [PATCH 144/265] [ISSUE #8669] Fix crc 32 overflow when lmq --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index f707d8fbd87..f34c6944c99 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -1834,12 +1834,13 @@ class DefaultAppendMessageCallback implements AppendMessageCallback { private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; // Store the message content private final ByteBuffer msgStoreItemMemory; - private final int crc32ReservedLength = enabledAppendPropCRC ? CommitLog.CRC32_RESERVED_LEN : 0; + private final int crc32ReservedLength; private final MessageStoreConfig messageStoreConfig; DefaultAppendMessageCallback(MessageStoreConfig messageStoreConfig) { this.msgStoreItemMemory = ByteBuffer.allocate(END_FILE_MIN_BLANK_LENGTH); this.messageStoreConfig = messageStoreConfig; + this.crc32ReservedLength = messageStoreConfig.isEnabledAppendPropCRC() ? CommitLog.CRC32_RESERVED_LEN : 0; } public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer, From 5eced111f445a319bfdc57e3d35cb015707f5dca Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Thu, 12 Sep 2024 16:46:31 +0800 Subject: [PATCH 145/265] [ISSUE #8259] fix parse ipv6 address in haproxy (#8260) * fix parse ipv6 from address in haproxy * more --- .../builder/DefaultAuthorizationContextBuilder.java | 8 ++++---- .../protocol/http2proxy/HAProxyMessageForwarder.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java index 02d5df236f5..e69abdaf805 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java @@ -171,7 +171,7 @@ public List build(ChannelHandlerContext context, Re subject = User.of(fields.get(SessionCredentials.ACCESS_KEY)); } String remoteAddr = RemotingHelper.parseChannelRemoteAddr(context.channel()); - String sourceIp = StringUtils.substringBefore(remoteAddr, CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON); Resource topic; Resource group; @@ -394,7 +394,7 @@ private List newContext(Metadata metadata, QueryRou subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } Resource resource = Resource.ofTopic(topic.getName()); - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Arrays.asList(Action.PUB, Action.SUB), sourceIp); return Collections.singletonList(context); } @@ -437,7 +437,7 @@ private static List newPubContext(Metadata metadata subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } Resource resource = Resource.ofTopic(topic.getName()); - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); DefaultAuthorizationContext context = DefaultAuthorizationContext.of(subject, resource, Action.PUB, sourceIp); return Collections.singletonList(context); } @@ -483,7 +483,7 @@ private static List newSubContexts(Metadata metadat if (metadata.containsKey(GrpcConstants.AUTHORIZATION_AK)) { subject = User.of(metadata.get(GrpcConstants.AUTHORIZATION_AK)); } - String sourceIp = StringUtils.substringBefore(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); + String sourceIp = StringUtils.substringBeforeLast(metadata.get(GrpcConstants.REMOTE_ADDRESS), CommonConstants.COLON); result.add(DefaultAuthorizationContext.of(subject, resource, Action.SUB, sourceIp)); return result; } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java index 39d7057bddd..518868831f4 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/protocol/http2proxy/HAProxyMessageForwarder.java @@ -118,11 +118,11 @@ protected HAProxyMessage buildHAProxyMessage(Channel inboundChannel) throws Ille } } else { String remoteAddr = RemotingHelper.parseChannelRemoteAddr(inboundChannel); - sourceAddress = StringUtils.substringBefore(remoteAddr, CommonConstants.COLON); + sourceAddress = StringUtils.substringBeforeLast(remoteAddr, CommonConstants.COLON); sourcePort = Integer.parseInt(StringUtils.substringAfterLast(remoteAddr, CommonConstants.COLON)); String localAddr = RemotingHelper.parseChannelLocalAddr(inboundChannel); - destinationAddress = StringUtils.substringBefore(localAddr, CommonConstants.COLON); + destinationAddress = StringUtils.substringBeforeLast(localAddr, CommonConstants.COLON); destinationPort = Integer.parseInt(StringUtils.substringAfterLast(localAddr, CommonConstants.COLON)); } From 00d50478614f55e088d7b2db1e9f5d167e1faaae Mon Sep 17 00:00:00 2001 From: imzs Date: Thu, 12 Sep 2024 18:07:52 +0800 Subject: [PATCH 146/265] [ISSUE #8688] fix typo, release the write lock in forceUpload() (#8689) From 7748040f41a69a7e18e2acb55c03005f539a4ba2 Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:45:21 +0800 Subject: [PATCH 147/265] [ISSUE #8599] Fix send fail with receiving GO_AWAY when rolling update proxy and add channel id in logs (#8685) --- .../remoting/netty/NettyRemotingAbstract.java | 8 +-- .../remoting/netty/NettyRemotingClient.java | 61 ++++++++++--------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 9f3136195b3..ffa37260594 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -39,8 +39,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.rocketmq.common.AbortProcessException; @@ -393,7 +393,7 @@ public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cm responseFuture.release(); } } else { - log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + log.warn("receive response, cmd={}, but not matched any request, address={}, channelId={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel()), ctx.channel().id()); } } @@ -560,13 +560,13 @@ public void operationFail(Throwable throwable) { return; } requestFail(opaque); - log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); + log.warn("send a request command to channel <{}>, channelId={}, failed.", RemotingHelper.parseChannelRemoteAddr(channel), channel.id()); }); return future; } catch (Exception e) { responseTable.remove(opaque); responseFuture.release(); - log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); + log.warn("send a request command to channel <{}> channelId={} Exception", RemotingHelper.parseChannelRemoteAddr(channel), channel.id(), e); future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); return future; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 41976122b2f..ef9762ddc67 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -49,7 +49,6 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.security.cert.CertificateException; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -416,14 +415,14 @@ public void closeChannel(final String addr, final Channel channel) { boolean removeItemFromTable = true; final ChannelWrapper prevCW = this.channelTables.get(addrRemote); - LOGGER.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, prevCW != null); + LOGGER.info("closeChannel: begin close the channel[addr={}, id={}] Found: {}", addrRemote, channel.id(), prevCW != null); if (null == prevCW) { - LOGGER.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] has been removed from the channel table before", addrRemote, channel.id()); removeItemFromTable = false; } else if (prevCW.isWrapperOf(channel)) { - LOGGER.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.", - addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] has been closed before, and has been created again, nothing to do.", + addrRemote, channel.id()); removeItemFromTable = false; } @@ -432,7 +431,7 @@ public void closeChannel(final String addr, final Channel channel) { if (channelWrapper != null && channelWrapper.tryClose(channel)) { this.channelTables.remove(addrRemote); } - LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id()); } RemotingHelper.closeChannel(channel); @@ -471,7 +470,7 @@ public void closeChannel(final Channel channel) { } if (null == prevCW) { - LOGGER.info("eventCloseChannel: the channel[{}] has been removed from the channel table before", addrRemote); + LOGGER.info("eventCloseChannel: the channel[addr={}, id={}] has been removed from the channel table before", RemotingHelper.parseChannelRemoteAddr(channel), channel.id()); removeItemFromTable = false; } @@ -480,11 +479,11 @@ public void closeChannel(final Channel channel) { if (channelWrapper != null && channelWrapper.tryClose(channel)) { this.channelTables.remove(addrRemote); } - LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + LOGGER.info("closeChannel: the channel[addr={}, id={}] was removed from channel table", addrRemote, channel.id()); RemotingHelper.closeChannel(channel); } } catch (Exception e) { - LOGGER.error("closeChannel: close the channel exception", e); + LOGGER.error("closeChannel: close the channel[id={}] exception", channel.id(), e); } finally { this.lockChannelTables.unlock(); } @@ -562,9 +561,9 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo boolean shouldClose = left > MIN_CLOSE_TIMEOUT_MILLIS || left > timeoutMillis / 4; if (nettyClientConfig.isClientCloseSocketIfTimeout() && shouldClose) { this.closeChannel(addr, channel); - LOGGER.warn("invokeSync: close socket because of timeout, {}ms, {}", timeoutMillis, channelRemoteAddr); + LOGGER.warn("invokeSync: close socket because of timeout, {}ms, channel[addr={}, id={}]", timeoutMillis, channelRemoteAddr, channel.id()); } - LOGGER.warn("invokeSync: wait response timeout exception, the channel[{}]", channelRemoteAddr); + LOGGER.warn("invokeSync: wait response timeout exception, the channel[addr={}, id={}]", channelRemoteAddr, channel.id()); throw e; } } else { @@ -819,10 +818,11 @@ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand response = responseFuture.getResponseCommand(); if (response.getCode() == ResponseCode.GO_AWAY) { if (nettyClientConfig.isEnableReconnectForGoAway()) { + LOGGER.info("Receive go away from channelId={}, channel={}", channel.id(), channel); ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { try { - if (channelWrapper0.reconnect()) { - LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); + if (channelWrapper0.reconnect(channel0)) { + LOGGER.info("Receive go away from channelId={}, channel={}, recreate the channelId={}", channel0.id(), channel0, channelWrapper0.getChannel().id()); channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); } } catch (Throwable t) { @@ -830,10 +830,11 @@ public CompletableFuture invokeImpl(final Channel channel, final } return channelWrapper0; }); - if (channelWrapper != null) { + if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) { if (nettyClientConfig.isEnableTransparentRetry()) { RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); retryRequest.setBody(request.getBody()); + retryRequest.setExtFields(request.getExtFields()); if (channelWrapper.isOK()) { long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); stopwatch.stop(); @@ -865,6 +866,8 @@ public CompletableFuture invokeImpl(final Channel channel, final return future; } } + } else { + LOGGER.warn("invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}", channel.id()); } } } @@ -1002,7 +1005,6 @@ class ChannelWrapper { // only affected by sync or async request, oneway is not included. private ChannelFuture channelToClose; private long lastResponseTime; - private volatile long lastReconnectTimestamp = 0L; private final String channelAddress; public ChannelWrapper(String address, ChannelFuture channelFuture) { @@ -1021,10 +1023,7 @@ public boolean isWritable() { } public boolean isWrapperOf(Channel channel) { - if (this.channelFuture.channel() != null && this.channelFuture.channel() == channel) { - return true; - } - return false; + return this.channelFuture.channel() != null && this.channelFuture.channel() == channel; } private Channel getChannel() { @@ -1052,20 +1051,27 @@ public String getChannelAddress() { return channelAddress; } - public boolean reconnect() { + public boolean reconnect(Channel channel) { + if (!isWrapperOf(channel)) { + LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id()); + return false; + } if (lock.writeLock().tryLock()) { try { - if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { + if (isWrapperOf(channel)) { channelToClose = channelFuture; String[] hostAndPort = getHostAndPort(channelAddress); channelFuture = fetchBootstrap(channelAddress) .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); - lastReconnectTimestamp = System.currentTimeMillis(); return true; + } else { + LOGGER.warn("channelWrapper has reconnect, so do nothing, now channelId={}, input channelId={}",getChannel().id(), channel.id()); } } finally { lock.writeLock().unlock(); } + } else { + LOGGER.warn("channelWrapper reconnect try lock fail, now channelId={}", getChannel().id()); } return false; } @@ -1152,7 +1158,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: ACTIVE, {}, channelId={}", remoteAddress, ctx.channel().id()); super.channelActive(ctx); if (NettyRemotingClient.this.channelEventListener != null) { @@ -1175,7 +1181,7 @@ public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws @Override public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: CLOSE channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); super.close(ctx, promise); NettyRemotingClient.this.failFast(ctx.channel()); @@ -1187,7 +1193,7 @@ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exce @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[{}]", remoteAddress); + LOGGER.info("NETTY CLIENT PIPELINE: channelInactive, the channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); super.channelInactive(ctx); } @@ -1198,7 +1204,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc IdleStateEvent event = (IdleStateEvent) evt; if (event.state().equals(IdleState.ALL_IDLE)) { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress); + LOGGER.warn("NETTY CLIENT PIPELINE: IDLE exception channel[addr={}, id={}]", remoteAddress, ctx.channel().id()); closeChannel(ctx.channel()); if (NettyRemotingClient.this.channelEventListener != null) { NettyRemotingClient.this @@ -1213,8 +1219,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); - LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress); - LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause); + LOGGER.warn("NETTY CLIENT PIPELINE: exceptionCaught channel[addr={}, id={}]", remoteAddress, ctx.channel().id(), cause); closeChannel(ctx.channel()); if (NettyRemotingClient.this.channelEventListener != null) { NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress, ctx.channel())); From 57d9f8d65301f40caf740fc8582b8b8929b73e83 Mon Sep 17 00:00:00 2001 From: yx9o Date: Sat, 14 Sep 2024 17:40:04 +0800 Subject: [PATCH 148/265] [ISSUE #8691] Fix PR-CI errors (#8692) * [ISSUE #8691] Fix PR-CI errors * Upgrade all actions/upload-artifact@v3 to actions/upload-artifact@v4 --- .github/workflows/pr-ci.yml | 4 ++-- .github/workflows/pr-e2e-test.yml | 8 ++++---- .github/workflows/push-ci.yml | 10 +++++----- .github/workflows/snapshot-automation.yml | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index ef2db755d00..99d7309fd0c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -21,7 +21,7 @@ jobs: - name: Build distribution tar run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -30,7 +30,7 @@ jobs: run: | mkdir -p ./pr echo ${{ github.event.number }} > ./pr/NR - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: pr path: pr/ diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index f9bb3bde75a..b1ff83eec39 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -68,7 +68,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -158,7 +158,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -199,7 +199,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -235,7 +235,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index 2fe62dbeb06..a522241a0ac 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -31,7 +31,7 @@ jobs: - name: Build distribution tar run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -72,7 +72,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -163,7 +163,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -204,7 +204,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: @@ -240,7 +240,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml index 99855d3aa0d..63b19417fe0 100644 --- a/.github/workflows/snapshot-automation.yml +++ b/.github/workflows/snapshot-automation.yml @@ -69,7 +69,7 @@ jobs: MAVEN_SETTINGS: ${{ github.workspace }}/.github/asf-deploy-settings.xml run: | mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: rocketmq @@ -110,7 +110,7 @@ jobs: mkdir versionlist touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`" sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Upload distribution tar with: name: versionlist @@ -200,7 +200,7 @@ jobs: annotate_only: true include_passed: true detailed_summary: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() name: Upload test log with: From f872b682e60651c7700247ec37958a06716d7f00 Mon Sep 17 00:00:00 2001 From: luozongle01 <150705370+luozongle01@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:40:37 +0800 Subject: [PATCH 149/265] [ISSUE #8695] fix DefaultLitePullConsumer PullThreadNums Parameter not effective. (#8696) --- .../impl/consumer/DefaultLitePullConsumerImpl.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index a3276cd7823..3f90b67ec99 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -164,10 +164,6 @@ private enum SubscriptionType { public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; - this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - this.defaultLitePullConsumer.getPullThreadNums(), - new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) - ); this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorMessageQueueChangeThread")); this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } @@ -293,6 +289,8 @@ public synchronized void start() throws MQClientException { this.defaultLitePullConsumer.changeInstanceNameToPID(); } + initScheduledThreadPoolExecutor(); + initMQClientFactory(); initRebalanceImpl(); @@ -324,6 +322,13 @@ public synchronized void start() throws MQClientException { } } + private void initScheduledThreadPoolExecutor() { + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.defaultLitePullConsumer.getPullThreadNums(), + new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) + ); + } + private void initMQClientFactory() throws MQClientException { this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); From d3e5f70979f74349db9c5ebbe3b48ff59b52d677 Mon Sep 17 00:00:00 2001 From: kaikoo <42601684+kingkh1995@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:42:46 +0800 Subject: [PATCH 150/265] [ISSUE #8613] fix start failed when acl2.0 authentication enabled (#8614) --- .../auth/authorization/factory/AuthorizationFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java index f87a5304cb7..29748a9ed44 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/factory/AuthorizationFactory.java @@ -105,7 +105,7 @@ public static AuthorizationEvaluator getEvaluator(AuthConfig config, Supplier public static AuthorizationStrategy getStrategy(AuthConfig config, Supplier metadataService) { try { Class clazz = StatelessAuthorizationStrategy.class; - if (StringUtils.isNotBlank(config.getAuthenticationStrategy())) { + if (StringUtils.isNotBlank(config.getAuthorizationStrategy())) { clazz = (Class) Class.forName(config.getAuthorizationStrategy()); } return clazz.getDeclaredConstructor(AuthConfig.class, Supplier.class).newInstance(config, metadataService); From a28c2cbc7d7e112830e5e0bf65a87a1ff50be570 Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:44:35 +0800 Subject: [PATCH 151/265] [RIP-70-1] Optimize the back pressure mechanism of the client (#8661) * Fix semaphore exception that failed halfway in the case of asynchronous message sending back pressure * Fix semaphore exception that failed halfway in the case of asynchronous message sending back pressure * Fix semaphore exception that failed halfway in the case of asynchronous message sending back pressure * Fixed variable typo * Fix semaphore exception that failed halfway in the case of asynchrono * Fix modifying the semaphore size at run time results in inaccurate semaphore calculations * optimize lock and introduce new lock * optimize code typo * Fix runtime modifications to semaphoreAsyncSendNum and semaphoreAsyncSendSize result in inaccurate actual semaphore values * fix fail test * adjust code typo * increase test * increase test * fix test * fix test --- .../impl/producer/DefaultMQProducerImpl.java | 71 +++++++++++++++---- .../client/lock/ReadWriteCASLock.java | 63 ++++++++++++++++ .../client/producer/DefaultMQProducer.java | 61 +++++++++++++++- .../producer/DefaultMQProducerTest.java | 44 ++++++++++++ 4 files changed, 222 insertions(+), 17 deletions(-) create mode 100644 client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 0e70ee25951..74a2516174a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -194,6 +194,14 @@ public void setSemaphoreAsyncSendSize(int size) { semaphoreAsyncSendSize = new Semaphore(size, true); } + public int getSemaphoreAsyncSendNumAvailablePermits() { + return semaphoreAsyncSendNum == null ? 0 : semaphoreAsyncSendNum.availablePermits(); + } + + public int getSemaphoreAsyncSendSizeAvailablePermits() { + return semaphoreAsyncSendSize == null ? 0 : semaphoreAsyncSendSize.availablePermits(); + } + public void initTransactionEnv() { TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; if (producer.getExecutorService() != null) { @@ -563,7 +571,7 @@ public void run() { class BackpressureSendCallBack implements SendCallback { public boolean isSemaphoreAsyncSizeAcquired = false; - public boolean isSemaphoreAsyncNumbAcquired = false; + public boolean isSemaphoreAsyncNumAcquired = false; public int msgLen; private final SendCallback sendCallback; @@ -573,24 +581,49 @@ public BackpressureSendCallBack(final SendCallback sendCallback) { @Override public void onSuccess(SendResult sendResult) { - if (isSemaphoreAsyncSizeAcquired) { - semaphoreAsyncSendSize.release(msgLen); - } - if (isSemaphoreAsyncNumbAcquired) { - semaphoreAsyncSendNum.release(); - } + semaphoreProcessor(); sendCallback.onSuccess(sendResult); } @Override public void onException(Throwable e) { + semaphoreProcessor(); + sendCallback.onException(e); + } + + public void semaphoreProcessor() { if (isSemaphoreAsyncSizeAcquired) { + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); semaphoreAsyncSendSize.release(msgLen); + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); } - if (isSemaphoreAsyncNumbAcquired) { + if (isSemaphoreAsyncNumAcquired) { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); semaphoreAsyncSendNum.release(); + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); } - sendCallback.onException(e); + } + + public void semaphoreAsyncAdjust(int semaphoreAsyncNum, int semaphoreAsyncSize) throws InterruptedException { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); + if (semaphoreAsyncNum > 0) { + semaphoreAsyncSendNum.release(semaphoreAsyncNum); + } else { + semaphoreAsyncSendNum.acquire(- semaphoreAsyncNum); + } + defaultMQProducer.setBackPressureForAsyncSendNumInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendNum() + + semaphoreAsyncNum); + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); + + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); + if (semaphoreAsyncSize > 0) { + semaphoreAsyncSendSize.release(semaphoreAsyncSize); + } else { + semaphoreAsyncSendSize.acquire(- semaphoreAsyncSize); + } + defaultMQProducer.setBackPressureForAsyncSendSizeInsideAdjust(defaultMQProducer.getBackPressureForAsyncSendSize() + + semaphoreAsyncSize); + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); } } @@ -599,32 +632,40 @@ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final throws MQClientException, InterruptedException { ExecutorService executor = this.getAsyncSenderExecutor(); boolean isEnableBackpressureForAsyncMode = this.getDefaultMQProducer().isEnableBackpressureForAsyncMode(); - boolean isSemaphoreAsyncNumbAcquired = false; + boolean isSemaphoreAsyncNumAcquired = false; boolean isSemaphoreAsyncSizeAcquired = false; int msgLen = msg.getBody() == null ? 1 : msg.getBody().length; + sendCallback.msgLen = msgLen; try { if (isEnableBackpressureForAsyncMode) { + defaultMQProducer.acquireBackPressureForAsyncSendNumLock(); long costTime = System.currentTimeMillis() - beginStartTime; - isSemaphoreAsyncNumbAcquired = timeout - costTime > 0 + + isSemaphoreAsyncNumAcquired = timeout - costTime > 0 && semaphoreAsyncSendNum.tryAcquire(timeout - costTime, TimeUnit.MILLISECONDS); - if (!isSemaphoreAsyncNumbAcquired) { + sendCallback.isSemaphoreAsyncNumAcquired = isSemaphoreAsyncNumAcquired; + defaultMQProducer.releaseBackPressureForAsyncSendNumLock(); + if (!isSemaphoreAsyncNumAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncNum timeout")); return; } + + defaultMQProducer.acquireBackPressureForAsyncSendSizeLock(); costTime = System.currentTimeMillis() - beginStartTime; + isSemaphoreAsyncSizeAcquired = timeout - costTime > 0 && semaphoreAsyncSendSize.tryAcquire(msgLen, timeout - costTime, TimeUnit.MILLISECONDS); + sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired; + defaultMQProducer.releaseBackPressureForAsyncSendSizeLock(); if (!isSemaphoreAsyncSizeAcquired) { sendCallback.onException( new RemotingTooMuchRequestException("send message tryAcquire semaphoreAsyncSize timeout")); return; } } - sendCallback.isSemaphoreAsyncSizeAcquired = isSemaphoreAsyncSizeAcquired; - sendCallback.isSemaphoreAsyncNumbAcquired = isSemaphoreAsyncNumbAcquired; - sendCallback.msgLen = msgLen; + executor.submit(runnable); } catch (RejectedExecutionException e) { if (isEnableBackpressureForAsyncMode) { diff --git a/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java new file mode 100644 index 00000000000..3d157313715 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/lock/ReadWriteCASLock.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.lock; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class ReadWriteCASLock { + //true : can lock ; false : not lock + private final AtomicBoolean writeLock = new AtomicBoolean(true); + + private final AtomicInteger readLock = new AtomicInteger(0); + + public void acquireWriteLock() { + boolean isLock = false; + do { + isLock = writeLock.compareAndSet(true, false); + } while (!isLock); + + do { + isLock = readLock.get() == 0; + } while (!isLock); + } + + public void releaseWriteLock() { + this.writeLock.compareAndSet(false, true); + } + + public void acquireReadLock() { + boolean isLock = false; + do { + isLock = writeLock.get(); + } while (!isLock); + readLock.getAndIncrement(); + } + + public void releaseReadLock() { + this.readLock.getAndDecrement(); + } + + public boolean getWriteLock() { + return this.writeLock.get() && this.readLock.get() == 0; + } + + public boolean getReadLock() { + return this.writeLock.get(); + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index b47c01f6764..f0842de8ba7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.lock.ReadWriteCASLock; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl; @@ -175,6 +176,16 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { private RPCHook rpcHook = null; + /** + * backPressureForAsyncSendNum is guaranteed to be modified at runtime and no new requests are allowed + */ + private final ReadWriteCASLock backPressureForAsyncSendNumLock = new ReadWriteCASLock(); + + /** + * backPressureForAsyncSendSize is guaranteed to be modified at runtime and no new requests are allowed + */ + private final ReadWriteCASLock backPressureForAsyncSendSizeLock = new ReadWriteCASLock(); + /** * Compress level of compress algorithm. */ @@ -1334,18 +1345,64 @@ public int getBackPressureForAsyncSendNum() { return backPressureForAsyncSendNum; } + /** + * For user modify backPressureForAsyncSendNum at runtime + */ public void setBackPressureForAsyncSendNum(int backPressureForAsyncSendNum) { + this.backPressureForAsyncSendNumLock.acquireWriteLock(); + backPressureForAsyncSendNum = Math.max(backPressureForAsyncSendNum, 10); + int acquiredBackPressureForAsyncSendNum = this.backPressureForAsyncSendNum + - defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits(); this.backPressureForAsyncSendNum = backPressureForAsyncSendNum; - defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum); + defaultMQProducerImpl.setSemaphoreAsyncSendNum(backPressureForAsyncSendNum - acquiredBackPressureForAsyncSendNum); + this.backPressureForAsyncSendNumLock.releaseWriteLock(); } public int getBackPressureForAsyncSendSize() { return backPressureForAsyncSendSize; } + /** + * For user modify backPressureForAsyncSendSize at runtime + */ public void setBackPressureForAsyncSendSize(int backPressureForAsyncSendSize) { + this.backPressureForAsyncSendSizeLock.acquireWriteLock(); + backPressureForAsyncSendSize = Math.max(backPressureForAsyncSendSize, 1024 * 1024); + int acquiredBackPressureForAsyncSendSize = this.backPressureForAsyncSendSize + - defaultMQProducerImpl.getSemaphoreAsyncSendSizeAvailablePermits(); + this.backPressureForAsyncSendSize = backPressureForAsyncSendSize; + defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize - acquiredBackPressureForAsyncSendSize); + this.backPressureForAsyncSendSizeLock.releaseWriteLock(); + } + + /** + * Used for system internal adjust backPressureForAsyncSendSize + */ + public void setBackPressureForAsyncSendSizeInsideAdjust(int backPressureForAsyncSendSize) { this.backPressureForAsyncSendSize = backPressureForAsyncSendSize; - defaultMQProducerImpl.setSemaphoreAsyncSendSize(backPressureForAsyncSendSize); + } + + /** + * Used for system internal adjust backPressureForAsyncSendNum + */ + public void setBackPressureForAsyncSendNumInsideAdjust(int backPressureForAsyncSendNum) { + this.backPressureForAsyncSendNum = backPressureForAsyncSendNum; + } + + public void acquireBackPressureForAsyncSendSizeLock() { + this.backPressureForAsyncSendSizeLock.acquireReadLock(); + } + + public void releaseBackPressureForAsyncSendSizeLock() { + this.backPressureForAsyncSendSizeLock.releaseReadLock(); + } + + public void acquireBackPressureForAsyncSendNumLock() { + this.backPressureForAsyncSendNumLock.acquireReadLock(); + } + + public void releaseBackPressureForAsyncSendNumLock() { + this.backPressureForAsyncSendNumLock.releaseReadLock(); } public List getTopics() { diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index be277f69bcf..4cf899f9708 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -551,6 +551,50 @@ public void testBatchSendMessageSync_Success() throws RemotingException, Interru producer.setAutoBatch(false); } + + @Test + public void testRunningSetBackCompress() throws RemotingException, InterruptedException, MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(5); + SendCallback sendCallback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + countDownLatch.countDown(); + } + + @Override + public void onException(Throwable e) { + e.printStackTrace(); + countDownLatch.countDown(); + } + }; + + // on enableBackpressureForAsyncMode + producer.setEnableBackpressureForAsyncMode(true); + producer.setBackPressureForAsyncSendNum(10); + producer.setBackPressureForAsyncSendSize(50 * 1024 * 1024); + Message message = new Message(); + message.setTopic("test"); + message.setBody("hello world".getBytes()); + MessageQueue mq = new MessageQueue("test", "BrokerA", 1); + //this message is send success + for (int i = 0; i < 5; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + producer.send(message, mq, sendCallback); + } catch (MQClientException | RemotingException | InterruptedException e) { + throw new RuntimeException(e); + } + } + }).start(); + } + producer.setBackPressureForAsyncSendNum(15); + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(producer.defaultMQProducerImpl.getSemaphoreAsyncSendNumAvailablePermits() + countDownLatch.getCount()).isEqualTo(15); + producer.setEnableBackpressureForAsyncMode(false); + } + public static TopicRouteData createTopicRoute() { TopicRouteData topicRouteData = new TopicRouteData(); From 280804c5592341f92a43b6d72ec6e94db77b74ac Mon Sep 17 00:00:00 2001 From: Liu Shengzhong Date: Wed, 18 Sep 2024 13:57:20 +0800 Subject: [PATCH 152/265] [ISSUE #8693] Fix checking MultiDispatchMessage when appending commitlog --- .../main/java/org/apache/rocketmq/store/CommitLog.java | 8 +++++--- .../java/org/apache/rocketmq/store/MessageExtEncoder.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index f34c6944c99..972e71aadd8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -61,6 +61,7 @@ import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; +import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -1903,7 +1904,7 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
    ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff(); - boolean isMultiDispatchMsg = messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner); + final boolean isMultiDispatchMsg = CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner); if (isMultiDispatchMsg) { AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner); if (appendMessageResult != null) { @@ -2244,8 +2245,9 @@ public FlushManager getFlushManager() { return flushManager; } - public static boolean isMultiDispatchMsg(MessageExtBrokerInner msg) { - return StringUtils.isNoneBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX); + public static boolean isMultiDispatchMsg(MessageStoreConfig messageStoreConfig, MessageExtBrokerInner msg) { + return StringUtils.isNotBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && + MultiDispatchUtils.isNeedHandleMultiDispatch(messageStoreConfig, msg.getTopic()); } private boolean isCloseReadAhead() { diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java index 20e9a652b7e..5c74918d9e6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java @@ -175,7 +175,7 @@ public PutMessageResult encodeWithoutProperties(MessageExtBrokerInner msgInner) public PutMessageResult encode(MessageExtBrokerInner msgInner) { this.byteBuf.clear(); - if (messageStoreConfig.isEnableMultiDispatch() && CommitLog.isMultiDispatchMsg(msgInner)) { + if (CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner)) { return encodeWithoutProperties(msgInner); } From d12635de6b920c237aaca4678fdb964fdda0fa18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Thu, 19 Sep 2024 13:50:05 +0800 Subject: [PATCH 153/265] [ISSUE #8707] Fix artifact download failure in CI after action upgrade (#8708) * Trigger push ci workflow * Bump @actions/download-artifact version to v4 * Revert "Trigger push ci workflow" This reverts commit 1d7f3a85528ee8d7f8b96d4c60a199a950b451b9. * Bump @actions/github-script to v7 * rerun ci --- .github/workflows/pr-e2e-test.yml | 5 +++-- .github/workflows/push-ci.yml | 4 ++-- .github/workflows/snapshot-automation.yml | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index b1ff83eec39..ead7103d603 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -25,7 +25,7 @@ jobs: java-version: ["8"] steps: - name: 'Download artifact' - uses: actions/github-script@v3.1.0 + uses: actions/github-script@v7 with: script: | var artifacts = await github.actions.listWorkflowRunArtifacts({ @@ -85,7 +85,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist @@ -96,6 +96,7 @@ jobs: a=(`ls versionlist`) printf '%s\n' "${a[@]}" | jq -R . | jq -s . echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT + deploy: if: ${{ success() }} name: Deploy RocketMQ diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index a522241a0ac..b23d69788cb 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -53,7 +53,7 @@ jobs: repository: apache/rocketmq-docker.git ref: master path: rocketmq-docker - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download distribution tar with: name: rocketmq @@ -90,7 +90,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml index 63b19417fe0..9fb16cb13ca 100644 --- a/.github/workflows/snapshot-automation.yml +++ b/.github/workflows/snapshot-automation.yml @@ -91,7 +91,7 @@ jobs: repository: apache/rocketmq-docker.git ref: master path: rocketmq-docker - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download distribution tar with: name: rocketmq @@ -125,7 +125,7 @@ jobs: outputs: version-json: ${{ steps.show_versions.outputs.version-json }} steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 name: Download versionlist with: name: versionlist From 3e81fae62a309521414398c607e589c2be49ee1e Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Thu, 19 Sep 2024 15:13:23 +0800 Subject: [PATCH 154/265] [ISSUE #8681] fix trace topic name (#8680) * fix trace topic --- .../client/trace/AsyncTraceDispatcher.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java index 6d62617eb8e..e321e1583d2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/AsyncTraceDispatcher.java @@ -302,14 +302,24 @@ public void run() { public void sendTraceData(List contextList) { Map> transBeanMap = new HashMap<>(16); - String currentRegionId; + String traceTopic; for (TraceContext context : contextList) { - currentRegionId = context.getRegionId(); + AccessChannel accessChannel = context.getAccessChannel(); + if (accessChannel == null) { + accessChannel = AsyncTraceDispatcher.this.accessChannel; + } + String currentRegionId = context.getRegionId(); if (currentRegionId == null || context.getTraceBeans().isEmpty()) { continue; } + if (AccessChannel.CLOUD == accessChannel) { + traceTopic = TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId; + } else { + traceTopic = traceTopicName; + } + String topic = context.getTraceBeans().get(0).getTopic(); - String key = topic + TraceConstants.CONTENT_SPLITOR + currentRegionId; + String key = topic + TraceConstants.CONTENT_SPLITOR + traceTopic; List transBeanList = transBeanMap.computeIfAbsent(key, k -> new ArrayList<>()); TraceTransferBean traceData = TraceDataEncoder.encoderFromContextBean(context); transBeanList.add(traceData); @@ -320,7 +330,7 @@ public void sendTraceData(List contextList) { } } - private void flushData(List transBeanList, String topic, String currentRegionId) { + private void flushData(List transBeanList, String topic, String traceTopic) { if (transBeanList.size() == 0) { return; } @@ -332,14 +342,14 @@ private void flushData(List transBeanList, String topic, Stri buffer.append(bean.getTransData()); count++; if (buffer.length() >= traceProducer.getMaxMessageSize()) { - sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); + sendTraceDataByMQ(keySet, buffer.toString(), traceTopic); buffer.delete(0, buffer.length()); keySet.clear(); count = 0; } } if (count > 0) { - sendTraceDataByMQ(keySet, buffer.toString(), TraceConstants.TRACE_TOPIC_PREFIX + currentRegionId); + sendTraceDataByMQ(keySet, buffer.toString(), traceTopic); } transBeanList.clear(); } From 6a241477310acd2a14d352215aad01fe9eb2f81b Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Thu, 19 Sep 2024 17:27:49 +0800 Subject: [PATCH 155/265] [ISSUE #8671] Replace channel.attr() set() and get() with RemotingHelper --- .../rocketmq/remoting/netty/NettyRemotingServer.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 51f8b85009e..8a329a37ac9 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -782,16 +782,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception private void handleWithMessage(HAProxyMessage msg, Channel channel) { try { if (StringUtils.isNotBlank(msg.sourceAddress())) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_ADDR).set(msg.sourceAddress()); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_ADDR, msg.sourceAddress()); } if (msg.sourcePort() > 0) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_PORT).set(String.valueOf(msg.sourcePort())); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_PORT, String.valueOf(msg.sourcePort())); } if (StringUtils.isNotBlank(msg.destinationAddress())) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR).set(msg.destinationAddress()); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_ADDR, msg.destinationAddress()); } if (msg.destinationPort() > 0) { - channel.attr(AttributeKeys.PROXY_PROTOCOL_SERVER_PORT).set(String.valueOf(msg.destinationPort())); + RemotingHelper.setPropertyToAttr(channel, AttributeKeys.PROXY_PROTOCOL_SERVER_PORT, String.valueOf(msg.destinationPort())); } if (CollectionUtils.isNotEmpty(msg.tlvs())) { msg.tlvs().forEach(tlv -> { @@ -811,6 +811,6 @@ protected void handleHAProxyTLV(HAProxyTLV tlv, Channel channel) { } AttributeKey key = AttributeKeys.valueOf( HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); - channel.attr(key).set(new String(valueBytes, CharsetUtil.UTF_8)); + RemotingHelper.setPropertyToAttr(channel, key, new String(valueBytes, CharsetUtil.UTF_8)); } } From 6024db77f59aa5810d47914edea9b8f0ae68b693 Mon Sep 17 00:00:00 2001 From: kissrain <370379624@qq.com> Date: Thu, 19 Sep 2024 17:28:53 +0800 Subject: [PATCH 156/265] Fix typo in BaseConf (#8679) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 赤远 --- .../src/test/java/org/apache/rocketmq/test/base/BaseConf.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index b64cda33420..472e106ce35 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -100,8 +100,8 @@ public class BaseConf { brokerController2.getBrokerConfig().getListenPort()); brokerController3 = IntegrationTestBase.createAndStartBroker(NAMESRV_ADDR); - log.debug("Broker {} started, listening: {}", brokerController2.getBrokerConfig().getBrokerName(), - brokerController2.getBrokerConfig().getListenPort()); + log.debug("Broker {} started, listening: {}", brokerController3.getBrokerConfig().getBrokerName(), + brokerController3.getBrokerConfig().getListenPort()); CLUSTER_NAME = brokerController1.getBrokerConfig().getBrokerClusterName(); BROKER1_NAME = brokerController1.getBrokerConfig().getBrokerName(); From 0d6c94be0e9ebeaa16acbaf6dd29f24e9349aa74 Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Thu, 19 Sep 2024 20:13:12 +0800 Subject: [PATCH 157/265] [ISSUE #8705] Make MQClientAPIFactory shutdown async (#8706) --- .../rocketmq/client/impl/MQClientAPIImpl.java | 5 +- .../impl/mqclient/MQClientAPIFactory.java | 5 +- .../common/utils/AsyncShutdownHelper.java | 76 +++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 8a3d3dd0dcb..b539b8f098a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -78,6 +78,7 @@ import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; @@ -184,9 +185,9 @@ import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetTopicsByClusterRequestHeader; import org.apache.rocketmq.remoting.protocol.header.GetUserRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListAclsRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ListUsersRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.HeartbeatRequestHeader; import org.apache.rocketmq.remoting.protocol.header.LockBatchMqRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader; @@ -247,7 +248,7 @@ import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; -public class MQClientAPIImpl implements NameServerUpdateCallback { +public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdown { private final static Logger log = LoggerFactory.getLogger(MQClientAPIImpl.class); private static boolean sendSmartMsg = Boolean.parseBoolean(System.getProperty("org.apache.rocketmq.client.sendSmartMsg", "true")); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java index c68859b2889..0fa31b66406 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIFactory.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.client.common.NameserverAccessConfig; import org.apache.rocketmq.client.impl.ClientRemotingProcessor; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.utils.AsyncShutdownHelper; import org.apache.rocketmq.common.utils.StartAndShutdown; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -85,9 +86,11 @@ public void start() throws Exception { @Override public void shutdown() throws Exception { + AsyncShutdownHelper helper = new AsyncShutdownHelper(); for (int i = 0; i < this.clientNum; i++) { - clients[i].shutdown(); + helper.addTarget(clients[i]); } + helper.shutdown().await(Integer.MAX_VALUE, TimeUnit.SECONDS); } protected MQClientAPIExt createAndStart(String instanceName) { diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java new file mode 100644 index 00000000000..da765d5e749 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/AsyncShutdownHelper.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class AsyncShutdownHelper { + private final AtomicBoolean shutdown; + private final List targetList; + + private CountDownLatch countDownLatch; + + public AsyncShutdownHelper() { + this.targetList = new ArrayList<>(); + this.shutdown = new AtomicBoolean(false); + } + + public void addTarget(Shutdown target) { + if (shutdown.get()) { + return; + } + targetList.add(target); + } + + public AsyncShutdownHelper shutdown() { + if (shutdown.get()) { + return this; + } + if (targetList.isEmpty()) { + return this; + } + this.countDownLatch = new CountDownLatch(targetList.size()); + for (Shutdown target : targetList) { + Runnable runnable = () -> { + try { + target.shutdown(); + } catch (Exception ignored) { + + } finally { + countDownLatch.countDown(); + } + }; + new Thread(runnable).start(); + } + return this; + } + + public boolean await(long time, TimeUnit unit) throws InterruptedException { + if (shutdown.get()) { + return false; + } + try { + return this.countDownLatch.await(time, unit); + } finally { + shutdown.compareAndSet(false, true); + } + } +} From cab4fdaec1186a03c0c909c124210b7c46dd20c5 Mon Sep 17 00:00:00 2001 From: rongtong Date: Fri, 20 Sep 2024 13:53:37 +0800 Subject: [PATCH 158/265] [ISSUE #8718] Fix flaky CreateAndUpdateTopicIT (#8717) --- .../test/route/CreateAndUpdateTopicIT.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java index 9004b91db39..9e9afb1ed2c 100644 --- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java @@ -17,13 +17,16 @@ package org.apache.rocketmq.test.route; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.util.MQAdminTestUtils; +import org.junit.Ignore; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; public class CreateAndUpdateTopicIT extends BaseConf { @@ -47,6 +50,8 @@ public void testCreateOrUpdateTopic_EnableSingleTopicRegistration() { } + // Temporarily ignore the fact that this test cannot pass in the integration test pipeline due to unknown reasons + @Ignore @Test public void testDeleteTopicFromNameSrvWithBrokerRegistration() { namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(true); @@ -60,11 +65,9 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() { boolean createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic1, 8, null); assertThat(createResult).isTrue(); - createResult = MQAdminTestUtils.createTopic(NAMESRV_ADDR, CLUSTER_NAME, testTopic2, 8, null); assertThat(createResult).isTrue(); - TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); assertThat(route.getBrokerDatas()).hasSize(3); @@ -73,11 +76,13 @@ public void testDeleteTopicFromNameSrvWithBrokerRegistration() { // Deletion is lazy, trigger broker registration brokerController1.registerBrokerAll(false, false, true); - // The route info of testTopic2 will be removed from broker1 after the registration - route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); - assertThat(route.getBrokerDatas()).hasSize(2); - assertThat(route.getQueueDatas().get(0).getBrokerName()).isEqualTo(BROKER2_NAME); - assertThat(route.getQueueDatas().get(1).getBrokerName()).isEqualTo(BROKER3_NAME); + await().atMost(10, TimeUnit.SECONDS).until(() -> { + // The route info of testTopic2 will be removed from broker1 after the registration + TopicRouteData finalRoute = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic2); + return finalRoute.getBrokerDatas().size() == 2 + && finalRoute.getQueueDatas().get(0).getBrokerName().equals(BROKER2_NAME) + && finalRoute.getQueueDatas().get(1).getBrokerName().equals(BROKER3_NAME); + }); brokerController1.getBrokerConfig().setEnableSingleTopicRegister(false); brokerController2.getBrokerConfig().setEnableSingleTopicRegister(false); From 845f468129b2c4159bf0d5e5854f441b67e3c73c Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Fri, 20 Sep 2024 17:19:23 +0800 Subject: [PATCH 159/265] [ISSUE #8720] Support disable netty server worker group by config (#8721) --- .../rocketmq/remoting/netty/NettyRemotingServer.java | 5 +++-- .../rocketmq/remoting/netty/NettyServerConfig.java | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index 8a329a37ac9..cbf25c23c60 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -270,8 +270,9 @@ public void run(Timeout timeout) { */ protected ChannelPipeline configChannel(SocketChannel ch) { return ch.pipeline() - .addLast(defaultEventExecutorGroup, HANDSHAKE_HANDLER_NAME, new HandshakeHandler()) - .addLast(defaultEventExecutorGroup, + .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null, + HANDSHAKE_HANDLER_NAME, new HandshakeHandler()) + .addLast(nettyServerConfig.isServerNettyWorkerGroupEnable() ? defaultEventExecutorGroup : null, encoder, new NettyDecoder(), distributionHandler, diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java index 6564404b920..664dee8371c 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java @@ -36,6 +36,7 @@ public class NettyServerConfig implements Cloneable { private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark; private int writeBufferLowWaterMark = NettySystemConfig.writeBufferLowWaterMark; private int serverSocketBacklog = NettySystemConfig.socketBacklog; + private boolean serverNettyWorkerGroupEnable = true; private boolean serverPooledByteBufAllocatorEnable = true; private boolean enableShutdownGracefully = false; @@ -175,6 +176,14 @@ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { this.writeBufferHighWaterMark = writeBufferHighWaterMark; } + public boolean isServerNettyWorkerGroupEnable() { + return serverNettyWorkerGroupEnable; + } + + public void setServerNettyWorkerGroupEnable(boolean serverNettyWorkerGroupEnable) { + this.serverNettyWorkerGroupEnable = serverNettyWorkerGroupEnable; + } + public boolean isEnableShutdownGracefully() { return enableShutdownGracefully; } From 945e7eacb165c2884faabe1c0eddcc7a39719d15 Mon Sep 17 00:00:00 2001 From: yx9o Date: Fri, 20 Sep 2024 17:46:22 +0800 Subject: [PATCH 160/265] [ISSUE #8604] Fix doc typo (#8605) * [ISSUE #8604] Fix doc typo * [ISSUE #8604] Fix doc typo --- docs/cn/best_practice.md | 2 +- docs/en/Configuration_Client.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn/best_practice.md b/docs/cn/best_practice.md index 5cc5b37643f..36d6acff6bd 100755 --- a/docs/cn/best_practice.md +++ b/docs/cn/best_practice.md @@ -253,7 +253,7 @@ DefaultMQProducer、TransactionMQProducer、DefaultMQPushConsumer、DefaultMQPul | clientIP | 本机IP | 客户端本机IP地址，某些机器会发生无法识别客户端IP地址情况，需要应用在代码中强制指定 | | instanceName | DEFAULT | 客户端实例名称，客户端创建的多个Producer、Consumer实际是共用一个内部实例（这个实例包含网络连接、线程资源等） | | clientCallbackExecutorThreads | 4 | 通信层异步回调线程数 | -| pollNameServerInteval | 30000 | 轮询Name Server间隔时间，单位毫秒 | +| pollNameServerInterval | 30000 | 轮询Name Server间隔时间，单位毫秒 | | heartbeatBrokerInterval | 30000 | 向Broker发送心跳间隔时间，单位毫秒 | | persistConsumerOffsetInterval | 5000 | 持久化Consumer消费进度间隔时间，单位毫秒 | diff --git a/docs/en/Configuration_Client.md b/docs/en/Configuration_Client.md index 4d999b2feda..4679957af5a 100644 --- a/docs/en/Configuration_Client.md +++ b/docs/en/Configuration_Client.md @@ -48,7 +48,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | clientIP | local IP | Client local ip address, some machines will fail to recognize the client IP address, which needs to be enforced in the code | | instanceName | DEFAULT | Name of the client instance, Multiple producers and consumers created by the client actually share one internal instance (this instance contains network connection, thread resources, etc.). | | clientCallbackExecutorThreads | 4 | Number of communication layer asynchronous callback threads | -| pollNameServerInteval | 30000 | Polling the Name Server interval in milliseconds | +| pollNameServerInterval | 30000 | Polling the Name Server interval in milliseconds | | heartbeatBrokerInterval | 30000 | The heartbeat interval, in milliseconds, is sent to the Broker | | persistConsumerOffsetInterval | 5000 | The persistent Consumer consumes the progress interval in milliseconds | From e8d1472fb7a592ee605a7e7a12faef82e8c672fa Mon Sep 17 00:00:00 2001 From: qianye <37405937+qianye1001@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:28:42 +0800 Subject: [PATCH 161/265] [ISSUE #8712] Set namesrvAddrChoosed to null if choosed addr is not exist (#8713) --- .../rocketmq/remoting/netty/NettyRemotingClient.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index ef9762ddc67..ae82b09edaf 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -520,10 +520,11 @@ public void updateNameServerAddressList(List addrs) { this.namesrvAddrList.set(addrs); // should close the channel if choosed addr is not exist. - if (this.namesrvAddrChoosed.get() != null && !addrs.contains(this.namesrvAddrChoosed.get())) { - String namesrvAddr = this.namesrvAddrChoosed.get(); + String chosenNameServerAddr = this.namesrvAddrChoosed.get(); + if (chosenNameServerAddr != null && !addrs.contains(chosenNameServerAddr)) { + namesrvAddrChoosed.compareAndSet(chosenNameServerAddr, null); for (String addr : this.channelTables.keySet()) { - if (addr.contains(namesrvAddr)) { + if (addr.contains(chosenNameServerAddr)) { ChannelWrapper channelWrapper = this.channelTables.get(addr); if (channelWrapper != null) { channelWrapper.close(); From 525f877f3bddced2d85d99520fd600bcbbfe3c6d Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:24:15 +0800 Subject: [PATCH 162/265] [ISSUE #8589] Support file format CQ and json format offset in-place upgrade to rocksdb management (#8600) --- .../rocketmq/broker/BrokerController.java | 6 +- .../broker/offset/ConsumerOffsetManager.java | 20 +++ .../offset/RocksDBConsumerOffsetManager.java | 77 ++++++++- .../processor/AdminBrokerProcessor.java | 92 ++++++++++- .../RocksDBSubscriptionGroupManager.java | 36 ++-- .../SubscriptionGroupManager.java | 20 +++ .../topic/RocksDBTopicConfigManager.java | 26 +-- .../broker/topic/TopicConfigManager.java | 20 +++ .../RocksdbTransferOffsetAndCqTest.java | 154 ++++++++++++++++++ .../rocketmq/client/impl/MQClientAPIImpl.java | 15 ++ .../common/config/AbstractRocksDBStorage.java | 23 +-- .../remoting/protocol/RequestCode.java | 1 + ...eckRocksdbCqWriteProgressResponseBody.java | 35 ++++ ...ckRocksdbCqWriteProgressRequestHeader.java | 47 ++++++ .../rocketmq/store/DefaultMessageStore.java | 42 ++++- .../rocketmq/store/RocksDBMessageStore.java | 44 ++++- .../store/config/MessageStoreConfig.java | 31 ++++ .../apache/rocketmq/store/queue/CqUnit.java | 1 + .../store/queue/RocksDBConsumeQueue.java | 3 +- .../store/queue/RocksDBConsumeQueueStore.java | 10 +- .../tools/admin/DefaultMQAdminExt.java | 7 + .../tools/admin/DefaultMQAdminExtImpl.java | 7 + .../rocketmq/tools/admin/MQAdminExt.java | 3 + .../ExportMetadataInRocksDBCommand.java | 4 +- .../CheckRocksdbCqWriteProgressCommand.java | 97 +++++++++++ 25 files changed, 755 insertions(+), 66 deletions(-) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 22ac7fedf1c..aaf06caddf8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -18,7 +18,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import java.io.IOException; import java.net.InetSocketAddress; import java.util.AbstractMap; import java.util.ArrayList; @@ -789,6 +788,9 @@ public boolean initializeMessageStore() { defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); } else { defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); + if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) { + defaultMessageStore.enableRocksdbCQWrite(); + } } if (messageStoreConfig.isEnableDLegerCommitLog()) { @@ -812,7 +814,7 @@ public boolean initializeMessageStore() { this.timerMessageStore.registerEscapeBridgeHook(msg -> escapeBridge.putMessage(msg)); this.messageStore.setTimerMessageStore(this.timerMessageStore); } - } catch (IOException e) { + } catch (Exception e) { result = false; LOG.error("BrokerController#initialize: unexpected error occurs", e); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java index 21f20dde325..403324137cc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPathConfigHelper; import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; @@ -373,6 +374,25 @@ public void setDataVersion(DataVersion dataVersion) { this.dataVersion = dataVersion; } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class); + if (obj != null) { + this.dataVersion = obj.dataVersion; + } + LOG.info("load consumer offset dataVersion success,{},{} ", fileName, jsonString); + } + return true; + } catch (Exception e) { + LOG.error("load consumer offset dataVersion failed " + fileName, e); + return false; + } + } + public void removeOffset(final String group) { Iterator>> it = this.offsetTable.entrySet().iterator(); while (it.hasNext()) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java index de293fc4992..1e7cda71eed 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java @@ -16,26 +16,31 @@ */ package org.apache.rocketmq.broker.offset; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; import java.io.File; import java.util.Iterator; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; - import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.protocol.DataVersion; import org.rocksdb.WriteBatch; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; - public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + protected RocksDBConfigManager rocksDBConfigManager; public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(configFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); } @Override @@ -43,9 +48,47 @@ public boolean load() { if (!rocksDBConfigManager.init()) { return false; } - return this.rocksDBConfigManager.loadData(this::decodeOffset); + if (!loadDataVersion() || !loadConsumerOffset()) { + return false; + } + + return true; + } + + public boolean loadConsumerOffset() { + return this.rocksDBConfigManager.loadData(this::decodeOffset) && merge(); + } + + private boolean merge() { + if (!brokerController.getMessageStoreConfig().isTransferOffsetJsonToRocksdb()) { + log.info("the switch transferOffsetJsonToRocksdb is off, no merge offset operation is needed."); + return true; + } + if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { + log.info("consumerOffset json file does not exist, so skip merge"); + return true; + } + if (!super.loadDataVersion()) { + log.error("load json consumerOffset dataVersion error, startup will exit"); + return false; + } + + final DataVersion dataVersion = super.getDataVersion(); + final DataVersion kvDataVersion = this.getDataVersion(); + if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load json consumerOffset info failed, startup will exit"); + return false; + } + this.persist(); + this.getDataVersion().assignNewOne(dataVersion); + updateDataVersion(); + log.info("update offset from json, dataVersion:{}, offsetTable: {} ", this.getDataVersion(), JSON.toJSONString(this.getOffsetTable())); + } + return true; } + @Override public boolean stop() { return this.rocksDBConfigManager.stop(); @@ -69,8 +112,7 @@ protected void decodeOffset(final byte[] key, final byte[] body) { LOG.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable()); } - @Override - public String configFilePath() { + public String rocksdbConfigFilePath() { return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "consumerOffsets" + File.separator; } @@ -103,4 +145,23 @@ private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupN byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); writeBatch.put(keyBytes, valueBytes); } + + @Override + public boolean loadDataVersion() { + return this.rocksDBConfigManager.loadDataVersion(); + } + + @Override + public DataVersion getDataVersion() { + return rocksDBConfigManager.getKvDataVersion(); + } + + public void updateDataVersion() { + try { + rocksDBConfigManager.updateKvDataVersion(); + } catch (Exception e) { + log.error("update consumer offset dataVersion error", e); + throw new RuntimeException(e); + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 28bd2549145..863f16e515e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -18,9 +18,11 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import io.opentelemetry.api.common.Attributes; import java.io.UnsupportedEncodingException; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; @@ -38,7 +40,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import io.opentelemetry.api.common.Attributes; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.AccessValidator; @@ -69,6 +70,7 @@ import org.apache.rocketmq.common.LockCallback; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UnlockCallback; @@ -137,6 +139,7 @@ import org.apache.rocketmq.remoting.protocol.body.TopicList; import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; import org.apache.rocketmq.remoting.protocol.body.UserInfo; +import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader; @@ -209,6 +212,7 @@ import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; @@ -217,8 +221,9 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; import org.apache.rocketmq.store.timer.TimerMessageStore; import org.apache.rocketmq.store.util.LibC; -import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; + import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS; +import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; public class AdminBrokerProcessor implements NettyRequestProcessor { @@ -339,6 +344,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return fetchAllConsumeStatsInBroker(ctx, request); case RequestCode.QUERY_CONSUME_QUEUE: return queryConsumeQueue(ctx, request); + case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS: + return this.checkRocksdbCqWriteProgress(ctx, request); case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN: return this.updateAndGetGroupForbidden(ctx, request); case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG: @@ -458,6 +465,71 @@ private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, Re return response; } + private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class); + String requestTopic = requestHeader.getTopic(); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + + DefaultMessageStore messageStore = (DefaultMessageStore) brokerController.getMessageStore(); + RocksDBMessageStore rocksDBMessageStore = messageStore.getRocksDBMessageStore(); + if (!messageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", "rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid"))); + return response; + } + + ConcurrentMap> cqTable = messageStore.getConsumeQueueTable(); + StringBuilder diffResult = new StringBuilder("check success, all is ok!\n"); + try { + if (StringUtils.isNotBlank(requestTopic)) { + processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult,false); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); + return response; + } + for (Map.Entry> topicEntry : cqTable.entrySet()) { + String topic = topicEntry.getKey(); + processConsumeQueuesForTopic(topicEntry.getValue(), topic, rocksDBMessageStore, diffResult,true); + } + diffResult.append("check all topic successful, size:").append(cqTable.size()); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); + + } catch (Exception e) { + LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e); + response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", e.getMessage()))); + } + return response; + } + + private void processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean checkAll) { + for (Map.Entry queueEntry : queueMap.entrySet()) { + Integer queueId = queueEntry.getKey(); + ConsumeQueueInterface jsonCq = queueEntry.getValue(); + ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId); + if (!checkAll) { + String format = String.format("\n[topic: %s, queue: %s] \n kvEarliest : %s | kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ", + topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit()); + diffResult.append(format).append("\n"); + } + long maxFileOffsetInQueue = jsonCq.getMaxOffsetInQueue(); + long minOffsetInQueue = kvCq.getMinOffsetInQueue(); + for (long i = minOffsetInQueue; i < maxFileOffsetInQueue; i++) { + Pair fileCqUnit = jsonCq.getCqUnitAndStoreTime(i); + Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i); + if (fileCqUnit == null || kvCqUnit == null) { + diffResult.append(String.format("[topic: %s, queue: %s, offset: %s] \n kv : %s \n file: %s \n", + topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null")); + return; + } + if (!checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) { + String diffInfo = String.format("[topic:%s, queue: %s offset: %s] \n file: %s \n kv: %s", + topic, queueId, i, kvCqUnit.getObject1(), fileCqUnit.getObject1()); + LOGGER.error(diffInfo); + diffResult.append(diffInfo).append("\n"); + return; + } + } + } + } @Override public boolean rejectRequest() { return false; @@ -3305,4 +3377,20 @@ private boolean validateBlackListConfigExist(Properties properties) { } return false; } + + private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) { + if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) { + return false; + } + if (cqUnit1.getSize() != cqUnit2.getSize()) { + return false; + } + if (cqUnit1.getPos() != cqUnit2.getPos()) { + return false; + } + if (cqUnit1.getBatchNum() != cqUnit2.getBatchNum()) { + return false; + } + return cqUnit1.getTagsCode() == cqUnit2.getTagsCode(); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java index 7df72dbe686..5119f78672c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java @@ -19,6 +19,12 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiConsumer; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.UtilAll; @@ -27,13 +33,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.rocksdb.RocksIterator; -import java.io.File; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.BiConsumer; - public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { protected RocksDBConfigManager rocksDBConfigManager; @@ -79,28 +78,30 @@ public boolean loadForbidden(BiConsumer biConsumer) { private boolean merge() { if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("The switch is off, no merge operation is needed."); + log.info("the switch transferMetadataJsonToRocksdb is off, no merge subGroup operation is needed."); return true; } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { - log.info("json file and json back file not exist, so skip merge"); + log.info("subGroup json file does not exist, so skip merge"); return true; } - - if (!super.load()) { - log.error("load group and forbidden info from json file error, startup will exit"); + if (!super.loadDataVersion()) { + log.error("load json subGroup dataVersion error, startup will exit"); return false; } - - final ConcurrentMap groupTable = this.getSubscriptionGroupTable(); - final ConcurrentMap> forbiddenTable = this.getForbiddenTable(); final DataVersion dataVersion = super.getDataVersion(); final DataVersion kvDataVersion = this.getDataVersion(); if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load group and forbidden info from json file error, startup will exit"); + return false; + } + final ConcurrentMap groupTable = this.getSubscriptionGroupTable(); for (Map.Entry entry : groupTable.entrySet()) { putSubscriptionGroupConfig(entry.getValue()); log.info("import subscription config to rocksdb, group={}", entry.getValue()); } + final ConcurrentMap> forbiddenTable = this.getForbiddenTable(); for (Map.Entry> entry : forbiddenTable.entrySet()) { try { this.rocksDBConfigManager.updateForbidden(entry.getKey(), JSON.toJSONString(entry.getValue())); @@ -110,8 +111,10 @@ private boolean merge() { return false; } } - this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + this.getDataVersion().assignNewOne(dataVersion); updateDataVersion(); + } else { + log.info("dataVersion is not greater than kvDataVersion, no need to merge group metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion); } log.info("finish marge subscription config from json file and merge to rocksdb"); this.persist(); @@ -196,6 +199,7 @@ public void updateDataVersion() { try { rocksDBConfigManager.updateKvDataVersion(); } catch (Exception e) { + log.error("update group config dataVersion error", e); throw new RuntimeException(e); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index f2a7e0482b1..e6855ef9a2a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -334,6 +334,26 @@ public DataVersion getDataVersion() { return dataVersion; } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class); + if (obj != null) { + this.dataVersion.assignNewOne(obj.dataVersion); + this.printLoadDataWhenFirstBoot(obj); + log.info("load subGroup dataVersion success,{},{}", fileName, obj.dataVersion); + } + } + return true; + } catch (Exception e) { + log.error("load subGroup dataVersion failed" + fileName, e); + return false; + } + } + public void deleteSubscriptionGroupConfig(final String groupName) { SubscriptionGroupConfig old = removeSubscriptionGroupConfig(groupName); this.forbiddenTable.remove(groupName); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java index 2a89dd7e024..466e6416f98 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java @@ -18,6 +18,9 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; +import java.io.File; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; import org.apache.rocketmq.common.TopicConfig; @@ -25,10 +28,6 @@ import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.protocol.DataVersion; -import java.io.File; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - public class RocksDBTopicConfigManager extends TopicConfigManager { protected RocksDBConfigManager rocksDBConfigManager; @@ -60,29 +59,35 @@ public boolean loadDataVersion() { private boolean merge() { if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("The switch is off, no merge operation is needed."); + log.info("the switch transferMetadataJsonToRocksdb is off, no merge topic operation is needed."); return true; } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { - log.info("json file and json back file not exist, so skip merge"); + log.info("topic json file does not exist, so skip merge"); return true; } - if (!super.load()) { - log.error("load topic config from json file error, startup will exit"); + if (!super.loadDataVersion()) { + log.error("load json topic dataVersion error, startup will exit"); return false; } - final ConcurrentMap topicConfigTable = this.getTopicConfigTable(); final DataVersion dataVersion = super.getDataVersion(); final DataVersion kvDataVersion = this.getDataVersion(); if (dataVersion.getCounter().get() > kvDataVersion.getCounter().get()) { + if (!super.load()) { + log.error("load topic config from json file error, startup will exit"); + return false; + } + final ConcurrentMap topicConfigTable = this.getTopicConfigTable(); for (Map.Entry entry : topicConfigTable.entrySet()) { putTopicConfig(entry.getValue()); log.info("import topic config to rocksdb, topic={}", entry.getValue()); } - this.rocksDBConfigManager.getKvDataVersion().assignNewOne(dataVersion); + this.getDataVersion().assignNewOne(dataVersion); updateDataVersion(); + } else { + log.info("dataVersion is not greater than kvDataVersion, no need to merge topic metaData, dataVersion={}, kvDataVersion={}", dataVersion, kvDataVersion); } log.info("finish read topic config from json file and merge to rocksdb"); this.persist(); @@ -150,6 +155,7 @@ public void updateDataVersion() { try { rocksDBConfigManager.updateKvDataVersion(); } catch (Exception e) { + log.error("update topic config dataVersion error", e); throw new RuntimeException(e); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index eab2896b001..25d3218f2ab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -637,6 +637,26 @@ public String encode() { return encode(false); } + public boolean loadDataVersion() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + if (jsonString != null) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = + TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class); + if (topicConfigSerializeWrapper != null) { + this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion()); + log.info("load topic metadata dataVersion success {}, {}", fileName, topicConfigSerializeWrapper.getDataVersion()); + } + } + return true; + } catch (Exception e) { + log.error("load topic metadata dataVersion failed" + fileName, e); + return false; + } + } + @Override public String configFilePath() { return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java new file mode 100644 index 00000000000..b4800aec24e --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.offset; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.collections.MapUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.RocksDBMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; +import org.apache.rocketmq.store.queue.CqUnit; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.rocksdb.RocksDBException; + +@RunWith(MockitoJUnitRunner.class) +public class RocksdbTransferOffsetAndCqTest { + + private final String basePath = Paths.get(System.getProperty("user.home"), + "unit-test-store", UUID.randomUUID().toString().substring(0, 16).toUpperCase()).toString(); + + private final String topic = "topic"; + private final String group = "group"; + private final String clientHost = "clientHost"; + private final int queueId = 1; + + private RocksDBConsumerOffsetManager rocksdbConsumerOffsetManager; + + private ConsumerOffsetManager consumerOffsetManager; + + private DefaultMessageStore defaultMessageStore; + + @Mock + private BrokerController brokerController; + + @Before + public void init() throws IOException { + if (notToBeExecuted()) { + return; + } + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setConsumerOffsetUpdateVersionStep(10); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(basePath); + messageStoreConfig.setTransferOffsetJsonToRocksdb(true); + messageStoreConfig.setRocksdbCQDoubleWriteEnable(true); + Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + + defaultMessageStore = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager("aaa", true), null, + brokerConfig, new ConcurrentHashMap()); + defaultMessageStore.enableRocksdbCQWrite(); + defaultMessageStore.loadCheckPoint(); + + consumerOffsetManager = new ConsumerOffsetManager(brokerController); + consumerOffsetManager.load(); + + rocksdbConsumerOffsetManager = new RocksDBConsumerOffsetManager(brokerController); + } + + @Test + public void testTransferOffset() { + if (notToBeExecuted()) { + return; + } + + for (int i = 0; i < 200; i++) { + consumerOffsetManager.commitOffset(clientHost, group, topic, queueId, i); + } + + ConcurrentMap> offsetTable = consumerOffsetManager.getOffsetTable(); + ConcurrentMap map = offsetTable.get(topic + "@" + group); + Assert.assertTrue(MapUtils.isNotEmpty(map)); + + Long offset = map.get(queueId); + Assert.assertEquals(199L, (long) offset); + + long offsetDataVersion = consumerOffsetManager.getDataVersion().getCounter().get(); + Assert.assertEquals(20L, offsetDataVersion); + + consumerOffsetManager.persist(); + + boolean loadResult = rocksdbConsumerOffsetManager.load(); + Assert.assertTrue(loadResult); + + ConcurrentMap> rocksdbOffsetTable = rocksdbConsumerOffsetManager.getOffsetTable(); + + ConcurrentMap rocksdbMap = rocksdbOffsetTable.get(topic + "@" + group); + Assert.assertTrue(MapUtils.isNotEmpty(rocksdbMap)); + + Long aLong1 = rocksdbMap.get(queueId); + Assert.assertEquals(199L, (long) aLong1); + + long rocksdbOffset = rocksdbConsumerOffsetManager.getDataVersion().getCounter().get(); + Assert.assertEquals(21L, rocksdbOffset); + } + + @Test + public void testRocksdbCqWrite() throws RocksDBException { + if (notToBeExecuted()) { + return; + } + RocksDBMessageStore kvStore = defaultMessageStore.getRocksDBMessageStore(); + ConsumeQueueStoreInterface store = kvStore.getConsumeQueueStore(); + ConsumeQueueInterface rocksdbCq = defaultMessageStore.getRocksDBMessageStore().findConsumeQueue(topic, queueId); + ConsumeQueueInterface fileCq = defaultMessageStore.findConsumeQueue(topic, queueId); + for (int i = 0; i < 200; i++) { + DispatchRequest request = new DispatchRequest(topic, queueId, i, 200, 0, System.currentTimeMillis(), i, "", "", 0, 0, new HashMap<>()); + fileCq.putMessagePositionInfoWrapper(request); + store.putMessagePositionInfoWrapper(request); + } + Pair unit = rocksdbCq.getCqUnitAndStoreTime(100); + Pair unit1 = fileCq.getCqUnitAndStoreTime(100); + Assert.assertTrue(unit.getObject1().getPos() == unit1.getObject1().getPos()); + } + + private boolean notToBeExecuted() { + return MixAll.isMac(); + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index b539b8f098a..0a45f096235 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -113,6 +113,7 @@ import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; @@ -148,6 +149,7 @@ import org.apache.rocketmq.remoting.protocol.header.AddBrokerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader; import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; @@ -3017,6 +3019,19 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr, throw new MQClientException(response.getCode(), response.getRemark()); } + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + CheckRocksdbCqWriteProgressRequestHeader header = new CheckRocksdbCqWriteProgressRequestHeader(); + header.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, header); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (ResponseCode.SUCCESS == response.getCode()) { + return CheckRocksdbCqWriteProgressResponseBody.decode(response.getBody(), CheckRocksdbCqWriteProgressResponseBody.class); + } + throw new MQClientException(response.getCode(), response.getRemark()); + } + public void checkClientInBroker(final String brokerAddr, final String consumerGroup, final String clientId, final SubscriptionData subscriptionData, final long timeoutMillis) diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index f88b8e198bf..13522889bb3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -17,6 +17,15 @@ package org.apache.rocketmq.common.config; import com.google.common.collect.Maps; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; @@ -40,16 +49,6 @@ import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - import static org.rocksdb.RocksDB.NOT_FOUND; public abstract class AbstractRocksDBStorage { @@ -495,7 +494,9 @@ public void statRocksdb(Logger logger) { String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage"); logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, memtable: {}, blocksPinnedByIterator: {}", blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); - } catch (Exception ignored) { + } catch (Exception e) { + logger.error("statRocksdb Failed. {}", this.dbPath, e); + throw new RuntimeException(e); } } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index f45ff6fa484..cfc5cc22785 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -217,6 +217,7 @@ public class RequestCode { public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352; public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353; + public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354; public static final int LITE_PULL_MESSAGE = 361; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java new file mode 100644 index 00000000000..76719ac1a24 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class CheckRocksdbCqWriteProgressResponseBody extends RemotingSerializable { + + String diffResult; + + public String getDiffResult() { + return diffResult; + } + + public void setDiffResult(String diffResult) { + this.diffResult = diffResult; + } + + +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java new file mode 100644 index 00000000000..fee158b4976 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.header; + +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.action.RocketMQAction; +import org.apache.rocketmq.common.resource.ResourceType; +import org.apache.rocketmq.common.resource.RocketMQResource; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RequestCode; + +@RocketMQAction(value = RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, action = Action.GET) +public class CheckRocksdbCqWriteProgressRequestHeader implements CommandCustomHeader { + + @CFNotNull + @RocketMQResource(ResourceType.TOPIC) + private String topic; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 8f564d5bc14..8b46c7f5ce4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -163,11 +163,13 @@ public class DefaultMessageStore implements MessageStore { private volatile boolean shutdown = true; protected boolean notifyMessageArriveInBatch = false; - private StoreCheckpoint storeCheckpoint; + protected StoreCheckpoint storeCheckpoint; private TimerMessageStore timerMessageStore; private final LinkedList dispatcherList; + private RocksDBMessageStore rocksDBMessageStore; + private RandomAccessFile lockFile; private FileLock lock; @@ -354,12 +356,7 @@ public boolean load() { } if (result) { - this.storeCheckpoint = - new StoreCheckpoint( - StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); - this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset(); - setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset()); - + loadCheckPoint(); result = this.indexService.load(lastExitOK); this.recover(lastExitOK); LOGGER.info("message store recover end, and the max phy offset = {}", this.getMaxPhyOffset()); @@ -381,6 +378,14 @@ public boolean load() { return result; } + public void loadCheckPoint() throws IOException { + this.storeCheckpoint = + new StoreCheckpoint( + StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); + this.masterFlushedOffset = this.storeCheckpoint.getMasterFlushedOffset(); + setConfirmOffset(this.storeCheckpoint.getConfirmPhyOffset()); + } + /** * @throws Exception */ @@ -511,6 +516,10 @@ public void shutdown() { this.compactionService.shutdown(); } + if (messageStoreConfig.isRocksdbCQDoubleWriteEnable()) { + this.rocksDBMessageStore.consumeQueueStore.shutdown(); + } + this.flushConsumeQueueService.shutdown(); this.allocateMappedFileService.shutdown(); this.storeCheckpoint.flush(); @@ -3251,6 +3260,17 @@ public HARuntimeInfo getHARuntimeInfo() { } } + public void enableRocksdbCQWrite() { + try { + RocksDBMessageStore store = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, this.topicConfigTable); + this.rocksDBMessageStore = store; + store.loadAndStartConsumerServiceOnly(); + addDispatcher(store.getDispatcherBuildRocksdbConsumeQueue()); + } catch (Exception e) { + LOGGER.error("enableRocksdbCqWrite error", e); + } + } + public int getMaxDelayLevel() { return maxDelayLevel; } @@ -3338,4 +3358,12 @@ public boolean isTransientStorePoolEnable() { public long getReputFromOffset() { return this.reputMessageService.getReputFromOffset(); } + + public RocksDBMessageStore getRocksDBMessageStore() { + return this.rocksDBMessageStore; + } + + public ConsumeQueueStoreInterface getConsumeQueueStore() { + return consumeQueueStore; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 6141b778bf7..90df7aed596 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.store; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.Meter; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Supplier; - -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.metrics.Meter; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; @@ -39,6 +39,8 @@ public class RocksDBMessageStore extends DefaultMessageStore { + private CommitLogDispatcherBuildRocksdbConsumeQueue dispatcherBuildRocksdbConsumeQueue; + public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { @@ -178,4 +180,40 @@ public void initMetrics(Meter meter, Supplier attributesBuild // Also add some metrics for rocksdb's monitoring. RocksDBStoreMetricsManager.init(meter, attributesBuilderSupplier, this); } + + public CommitLogDispatcherBuildRocksdbConsumeQueue getDispatcherBuildRocksdbConsumeQueue() { + return dispatcherBuildRocksdbConsumeQueue; + } + + class CommitLogDispatcherBuildRocksdbConsumeQueue implements CommitLogDispatcher { + @Override + public void dispatch(DispatchRequest request) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + putMessagePositionInfo(request); + break; + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + } + } + } + + public void loadAndStartConsumerServiceOnly() { + try { + this.dispatcherBuildRocksdbConsumeQueue = new CommitLogDispatcherBuildRocksdbConsumeQueue(); + boolean loadResult = this.consumeQueueStore.load(); + if (!loadResult) { + throw new RuntimeException("load consume queue failed"); + } + super.loadCheckPoint(); + this.consumeQueueStore.start(); + } catch (Exception e) { + ERROR_LOG.error("loadAndStartConsumerServiceOnly error", e); + throw new RuntimeException(e); + } + } + } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 0b45d92418e..c077831f3c4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -424,6 +424,37 @@ public class MessageStoreConfig { private boolean putConsumeQueueDataByFileChannel = true; + private boolean transferOffsetJsonToRocksdb = false; + + private boolean rocksdbCQDoubleWriteEnable = false; + + private boolean enableBatchWriteKvCq = true; + + + public boolean isEnableBatchWriteKvCq() { + return enableBatchWriteKvCq; + } + + public void setEnableBatchWriteKvCq(boolean enableBatchWriteKvCq) { + this.enableBatchWriteKvCq = enableBatchWriteKvCq; + } + + public boolean isRocksdbCQDoubleWriteEnable() { + return rocksdbCQDoubleWriteEnable; + } + + public void setRocksdbCQDoubleWriteEnable(boolean rocksdbWriteEnable) { + this.rocksdbCQDoubleWriteEnable = rocksdbWriteEnable; + } + + public boolean isTransferOffsetJsonToRocksdb() { + return transferOffsetJsonToRocksdb; + } + + public void setTransferOffsetJsonToRocksdb(boolean transferOffsetJsonToRocksdb) { + this.transferOffsetJsonToRocksdb = transferOffsetJsonToRocksdb; + } + public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java index b8865fd9195..34f5cb142b6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/CqUnit.java @@ -109,6 +109,7 @@ public String toString() { ", size=" + size + ", pos=" + pos + ", batchNum=" + batchNum + + ", tagsCode=" + tagsCode + ", compactedOffset=" + compactedOffset + '}'; } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java index 5a981bb4df1..2363c2896e5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -18,7 +18,6 @@ import java.nio.ByteBuffer; import java.util.List; - import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.attribute.CQType; @@ -311,7 +310,7 @@ public CqUnit getEarliestUnit() { public CqUnit getLatestUnit() { try { long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); - return get(maxOffset); + return get(maxOffset > 0 ? maxOffset - 1 : maxOffset); } catch (RocksDBException e) { ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 3c6b91ec018..34c6d2f3956 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -28,7 +28,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; @@ -78,6 +77,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private final Map> tempTopicQueueMaxOffsetMap; private volatile boolean isCQError = false; + private boolean enableBatchWriteKvCq; + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); @@ -87,6 +88,7 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); this.writeBatch = new WriteBatch(); + this.enableBatchWriteKvCq = messageStoreConfig.isEnableBatchWriteKvCq(); this.bufferDRList = new ArrayList(BATCH_SIZE); this.cqBBPairList = new ArrayList(BATCH_SIZE); this.offsetBBPairList = new ArrayList(BATCH_SIZE); @@ -164,12 +166,12 @@ private boolean shutdownInner() { @Override public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { - if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { - putMessagePosition(); - } if (request != null) { this.bufferDRList.add(request); } + if (request == null || !enableBatchWriteKvCq || this.bufferDRList.size() >= BATCH_SIZE) { + putMessagePosition(); + } } public void putMessagePosition() throws RocksDBException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 6ebee1d0dd1..3686bf2644b 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -771,6 +772,12 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String ); } + @Override + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic); + } + @Override public boolean resumeCheckHalfMessage(String topic, String msgId) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index dc4d35e7049..883dcbe41d7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -90,6 +90,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -1817,6 +1818,12 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String return this.mqClientInstance.getMQClientAPIImpl().queryConsumeQueue(brokerAddr, topic, queueId, index, count, consumerGroup, timeoutMillis); } + @Override + public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl().checkRocksdbCqWriteProgress(brokerAddr, topic, timeoutMillis); + } + @Override public boolean resumeCheckHalfMessage(final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index ff78f22c704..09204ab7be2 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -148,6 +149,8 @@ ConsumeStats examineConsumeStats( final String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; + ConsumeStats examineConsumeStats(final String consumerGroup, final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java index 1ecb1fa2cd9..c466490b8a8 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.rocketmq.tools.command.export; import com.alibaba.fastjson.JSONObject; @@ -77,6 +78,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t } String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + path += "/" + configType; boolean jsonEnable = false; if (commandLine.hasOption("jsonEnable")) { @@ -86,7 +88,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); if (!kvStore.start()) { - System.out.print("RocksDB load error, path=" + path + "\n"); + System.out.printf("RocksDB load error, path=%s\n" , path); return; } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java new file mode 100644 index 00000000000..82dcb741962 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.tools.command.queue; + +import java.util.Map; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.SubCommand; + +public class CheckRocksdbCqWriteProgressCommand implements SubCommand { + + @Override + public String commandName() { + return "checkRocksdbCqWriteProgressCommandCommand"; + } + + @Override + public String commandDesc() { + return "check if rocksdb cq is same as file cq"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("c", "cluster", true, "cluster name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("n", "nameserverAddr", true, "nameserverAddr"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + defaultMQAdminExt.setNamesrvAddr(StringUtils.trim(commandLine.getOptionValue('n'))); + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : ""; + String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : ""; + + try { + defaultMQAdminExt.start(); + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + Map> clusterAddrTable = clusterInfo.getClusterAddrTable(); + Map brokerAddrTable = clusterInfo.getBrokerAddrTable(); + if (clusterAddrTable.get(clusterName) == null) { + System.out.print("clusterAddrTable is empty"); + return; + } + for (Map.Entry entry : brokerAddrTable.entrySet()) { + String brokerName = entry.getKey(); + BrokerData brokerData = entry.getValue(); + String brokerAddr = brokerData.getBrokerAddrs().get(0L); + CheckRocksdbCqWriteProgressResponseBody body = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic); + if (StringUtils.isNotBlank(topic)) { + System.out.printf(body.getDiffResult()); + } else { + System.out.printf(brokerName + " | " + brokerAddr + " | " + body.getDiffResult()); + } + } + + } catch (Exception e) { + throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} From a8dde86295a1efc67dbe74ee78a6374f0db2de70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Wed, 25 Sep 2024 10:08:42 +0800 Subject: [PATCH 163/265] [ISSUE #8742] Enhance unit test retry mechanism to trigger on PR submission (#8741) * Enable retry mechanism when submitting PR * Update * Remove gh run watch * Fix bug * Revert "Remove gh run watch" This reverts commit cd5da59a1aa782ef38ee9ae399cb8e4e185efb94. * Add 'Build and Run Tests by Bazel' --- .github/workflows/bazel.yml | 15 +-------------- .github/workflows/maven.yaml | 16 +--------------- .github/workflows/rerun-workflow.yml | 17 +++++++++-------- 3 files changed, 11 insertions(+), 37 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5aa4f460c7c..510457ca46e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -8,9 +8,6 @@ on: - develop - bazel -permissions: - actions: write - jobs: build: name: "bazel-compile (${{ matrix.os }})" @@ -23,14 +20,4 @@ jobs: - name: Build run: bazel build --config=remote //... - name: Run Tests - run: bazel test --config=remote //... - - name: Retry if failed - # if it failed , retry 2 times at most - if: failure() && fromJSON(github.run_attempt) < 3 - continue-on-error: true - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Attempting to retry workflow..." - gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file + run: bazel test --config=remote //... \ No newline at end of file diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index f17c20b1ab8..d0c0ba7d9f1 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -5,9 +5,6 @@ on: push: branches: [master, develop, bazel] -permissions: - actions: write - jobs: java_build: name: "maven-compile (${{ matrix.os }}, JDK-${{ matrix.jdk }})" @@ -44,15 +41,4 @@ jobs: with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log - retention-days: 1 - - - name: Retry if failed - # if it failed , retry 2 times at most - if: failure() && fromJSON(github.run_attempt) < 3 - continue-on-error: true - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "Attempting to retry workflow..." - gh workflow run rerun-workflow.yml -F run_id=${{ github.run_id }} \ No newline at end of file + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/rerun-workflow.yml b/.github/workflows/rerun-workflow.yml index bf83fc51b63..6c319505d2c 100644 --- a/.github/workflows/rerun-workflow.yml +++ b/.github/workflows/rerun-workflow.yml @@ -1,21 +1,22 @@ name: Rerun workflow on: - workflow_dispatch: - inputs: - run_id: - required: true + workflow_run: + workflows: ["Build and Run Tests by Maven" , "Build and Run Tests by Bazel"] + types: + - completed permissions: actions: write jobs: rerun: + if: github.event.workflow_run.conclusion == 'failure' && fromJSON(github.event.workflow_run.run_attempt) < 3 runs-on: ubuntu-latest steps: - - name: rerun ${{ inputs.run_id }} + - name: rerun ${{ github.event.workflow_run.id }} env: - GH_REPO: ${{ github.repository }} + GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 - gh run rerun ${{ inputs.run_id }} --failed \ No newline at end of file + gh run watch ${{ github.event.workflow_run.id }} > /dev/null 2>&1 + gh run rerun ${{ github.event.workflow_run.id }} --failed \ No newline at end of file From e2abbc31d9feb59d04879071f3e123564374d9c4 Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Wed, 25 Sep 2024 10:14:38 +0800 Subject: [PATCH 164/265] [ISSUE #8740] fix rocksDBConfigToJson command (#8738) * fix rocksDBConfigToJson command * fix --- .../metadata/RocksDBConfigToJsonCommand.java | 70 +++++++++++++++---- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java index 1d81287ac7d..f2803b0cbb3 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.tools.command.metadata; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; @@ -33,10 +32,13 @@ import java.io.File; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; public class RocksDBConfigToJsonCommand implements SubCommand { private static final String TOPICS_JSON_CONFIG = "topics"; private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; + private static final String CONSUMER_OFFSETS_JSON_CONFIG = "consumerOffsets"; @Override public String commandName() { @@ -45,7 +47,7 @@ public String commandName() { @Override public String commandDesc() { - return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; + return "Convert RocksDB kv config (topics/subscriptionGroups/consumerOffsets) to json"; } @Override @@ -56,7 +58,7 @@ public Options buildCommandlineOptions(Options options) { options.addOption(pathOption); Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + - "topics/subscriptionGroups"); + "topics/subscriptionGroups/consumerOffsets"); configTypeOption.setRequired(true); options.addOption(configTypeOption); @@ -71,19 +73,21 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t return; } - String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); + String configType = commandLine.getOptionValue("configType").trim(); if (!path.endsWith("/")) { path += "/"; } path += configType; - + if (CONSUMER_OFFSETS_JSON_CONFIG.equalsIgnoreCase(configType)) { + printConsumerOffsets(path); + return; + } ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true); configRocksDBStorage.start(); RocksIterator iterator = configRocksDBStorage.iterator(); - try { final Map configMap = new HashMap<>(); - final Map configTable = new HashMap<>(); + final JSONObject configTable = new JSONObject(); iterator.seekToFirst(); while (iterator.isValid()) { final byte[] key = iterator.key(); @@ -95,14 +99,16 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t iterator.next(); } byte[] kvDataVersion = configRocksDBStorage.getKvDataVersion(); - configMap.put("dataVersion", - JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8))); + if (kvDataVersion != null) { + configMap.put("dataVersion", + JSONObject.parseObject(new String(kvDataVersion, DataConverter.CHARSET_UTF8))); + } - if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { - configMap.put("topicConfigTable", JSON.parseObject(JSONObject.toJSONString(configTable))); + if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType)) { + configMap.put("topicConfigTable", configTable); } - if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { - configMap.put("subscriptionGroupTable", JSON.parseObject(JSONObject.toJSONString(configTable))); + if (SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { + configMap.put("subscriptionGroupTable", configTable); } System.out.print(JSONObject.toJSONString(configMap, true) + "\n"); } catch (Exception e) { @@ -111,4 +117,42 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t configRocksDBStorage.shutdown(); } } + + private void printConsumerOffsets(String path) { + ConfigRocksDBStorage configRocksDBStorage = new ConfigRocksDBStorage(path, true); + configRocksDBStorage.start(); + RocksIterator iterator = configRocksDBStorage.iterator(); + try { + final Map configMap = new HashMap<>(); + final JSONObject configTable = new JSONObject(); + iterator.seekToFirst(); + while (iterator.isValid()) { + final byte[] key = iterator.key(); + final byte[] value = iterator.value(); + final String name = new String(key, DataConverter.CHARSET_UTF8); + final String config = new String(value, DataConverter.CHARSET_UTF8); + final RocksDBOffsetSerializeWrapper jsonObject = JSONObject.parseObject(config, RocksDBOffsetSerializeWrapper.class); + configTable.put(name, jsonObject.getOffsetTable()); + iterator.next(); + } + configMap.put("offsetTable", configTable); + System.out.print(JSONObject.toJSONString(configMap, true) + "\n"); + } catch (Exception e) { + System.out.print("Error occurred while converting RocksDB kv config to json, " + "configType=consumerOffsets, " + e.getMessage() + "\n"); + } finally { + configRocksDBStorage.shutdown(); + } + } + + static class RocksDBOffsetSerializeWrapper { + private ConcurrentMap offsetTable = new ConcurrentHashMap<>(16); + + public ConcurrentMap getOffsetTable() { + return offsetTable; + } + + public void setOffsetTable(ConcurrentMap offsetTable) { + this.offsetTable = offsetTable; + } + } } \ No newline at end of file From 59bafe8c075668b0b386826f8de46f36f9c9192b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Wed, 25 Sep 2024 14:54:26 +0800 Subject: [PATCH 165/265] [ISSUE #8747] Fix PR E2E artifact download issue (#8748) * Update download artifact script * Trigger ci * Revert "Trigger ci" This reverts commit a971f329b1dd2e2927895bcac292bb19bc0b5cac. * Trigger ci --- .github/workflows/pr-e2e-test.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index ead7103d603..5b4264266ef 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -25,18 +25,18 @@ jobs: java-version: ["8"] steps: - name: 'Download artifact' - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ + let artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{github.event.workflow_run.id }}, }); - var matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => { + let matchArtifactRmq = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "rocketmq" })[0]; - var download = await github.actions.downloadArtifact({ + let download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifactRmq.id, @@ -259,5 +259,4 @@ jobs: action: "clean" ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" test-version: "${{ matrix.version }}" - job-id: ${{ strategy.job-index }} - + job-id: ${{ strategy.job-index }} \ No newline at end of file From 3b5cbf86df019b46ca6f422e343e8c4401d2db4c Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Wed, 25 Sep 2024 19:32:47 +0800 Subject: [PATCH 166/265] [ISSUE #8698] Remove batch write in kv cq store and update rocksdb cq check tool (#8739) --- .../processor/AdminBrokerProcessor.java | 24 ++++++++++++------- .../store/config/MessageStoreConfig.java | 10 ++++---- .../plugin/AbstractPluginMessageStore.java | 4 ++++ .../store/queue/RocksDBConsumeQueueStore.java | 21 ++++++++-------- .../tools/command/MQAdminStartup.java | 2 ++ .../CheckRocksdbCqWriteProgressCommand.java | 6 ++--- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 863f16e515e..80f3f44facb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -215,6 +215,7 @@ import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; @@ -470,16 +471,21 @@ private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, R String requestTopic = requestHeader.getTopic(); final RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - - DefaultMessageStore messageStore = (DefaultMessageStore) brokerController.getMessageStore(); - RocksDBMessageStore rocksDBMessageStore = messageStore.getRocksDBMessageStore(); - if (!messageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { + MessageStore messageStore = brokerController.getMessageStore(); + DefaultMessageStore defaultMessageStore; + if (messageStore instanceof AbstractPluginMessageStore) { + defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext(); + } else { + defaultMessageStore = (DefaultMessageStore) messageStore; + } + RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore(); + if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", "rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid"))); return response; } - ConcurrentMap> cqTable = messageStore.getConsumeQueueTable(); - StringBuilder diffResult = new StringBuilder("check success, all is ok!\n"); + ConcurrentMap> cqTable = defaultMessageStore.getConsumeQueueTable(); + StringBuilder diffResult = new StringBuilder(); try { if (StringUtils.isNotBlank(requestTopic)) { processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult,false); @@ -516,15 +522,15 @@ private void processConsumeQueuesForTopic(ConcurrentMap fileCqUnit = jsonCq.getCqUnitAndStoreTime(i); Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i); if (fileCqUnit == null || kvCqUnit == null) { - diffResult.append(String.format("[topic: %s, queue: %s, offset: %s] \n kv : %s \n file: %s \n", + diffResult.append(String.format("[topic: %s, queue: %s, offset: %s] \n kv : %s \n file : %s \n", topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null")); return; } if (!checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) { - String diffInfo = String.format("[topic:%s, queue: %s offset: %s] \n file: %s \n kv: %s", + String diffInfo = String.format("[topic:%s, queue: %s offset: %s] \n file : %s \n kv : %s \n", topic, queueId, i, kvCqUnit.getObject1(), fileCqUnit.getObject1()); LOGGER.error(diffInfo); - diffResult.append(diffInfo).append("\n"); + diffResult.append(diffInfo).append(System.lineSeparator()); return; } } diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index c077831f3c4..68531284389 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -428,15 +428,15 @@ public class MessageStoreConfig { private boolean rocksdbCQDoubleWriteEnable = false; - private boolean enableBatchWriteKvCq = true; + private int batchWriteKvCqSize = 16; - public boolean isEnableBatchWriteKvCq() { - return enableBatchWriteKvCq; + public int getBatchWriteKvCqSize() { + return batchWriteKvCqSize; } - public void setEnableBatchWriteKvCq(boolean enableBatchWriteKvCq) { - this.enableBatchWriteKvCq = enableBatchWriteKvCq; + public void setBatchWriteKvCqSize(int batchWriteKvCqSize) { + this.batchWriteKvCqSize = batchWriteKvCqSize; } public boolean isRocksdbCQDoubleWriteEnable() { diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 2f2ce981257..2401257c306 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -661,4 +661,8 @@ public void recoverTopicQueueTable() { public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { next.notifyMessageArriveIfNecessary(dispatchRequest); } + + public MessageStore getNext() { + return next; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 34c6d2f3956..c889ae7ca85 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -55,7 +55,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { public static final byte CTRL_1 = '\u0001'; public static final byte CTRL_2 = '\u0002'; - private static final int BATCH_SIZE = 16; + private final int batchSize; public static final int MAX_KEY_LEN = 300; private final ScheduledExecutorService scheduledExecutorService; @@ -77,8 +77,6 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private final Map> tempTopicQueueMaxOffsetMap; private volatile boolean isCQError = false; - private boolean enableBatchWriteKvCq; - public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); @@ -88,11 +86,11 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); this.writeBatch = new WriteBatch(); - this.enableBatchWriteKvCq = messageStoreConfig.isEnableBatchWriteKvCq(); - this.bufferDRList = new ArrayList(BATCH_SIZE); - this.cqBBPairList = new ArrayList(BATCH_SIZE); - this.offsetBBPairList = new ArrayList(BATCH_SIZE); - for (int i = 0; i < BATCH_SIZE; i++) { + this.batchSize = messageStoreConfig.getBatchWriteKvCqSize(); + this.bufferDRList = new ArrayList(batchSize); + this.cqBBPairList = new ArrayList(batchSize); + this.offsetBBPairList = new ArrayList(batchSize); + for (int i = 0; i < batchSize; i++) { this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); } @@ -166,12 +164,13 @@ private boolean shutdownInner() { @Override public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { + if (request == null || this.bufferDRList.size() >= batchSize) { + putMessagePosition(); + } + if (request != null) { this.bufferDRList.add(request); } - if (request == null || !enableBatchWriteKvCq || this.bufferDRList.size() >= BATCH_SIZE) { - putMessagePosition(); - } } public void putMessagePosition() throws RocksDBException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 43e4259c4e1..313a777ce4f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -104,6 +104,7 @@ import org.apache.rocketmq.tools.command.offset.ResetOffsetByTimeCommand; import org.apache.rocketmq.tools.command.offset.SkipAccumulationSubCommand; import org.apache.rocketmq.tools.command.producer.ProducerSubCommand; +import org.apache.rocketmq.tools.command.queue.CheckRocksdbCqWriteProgressCommand; import org.apache.rocketmq.tools.command.queue.QueryConsumeQueueCommand; import org.apache.rocketmq.tools.command.stats.StatsAllSubCommand; import org.apache.rocketmq.tools.command.topic.AllocateMQSubCommand; @@ -304,6 +305,7 @@ public static void initCommand() { initCommand(new ListAclSubCommand()); initCommand(new CopyAclsSubCommand()); initCommand(new RocksDBConfigToJsonCommand()); + initCommand(new CheckRocksdbCqWriteProgressCommand()); } private static void printHelp() { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java index 82dcb741962..d18a24ee1dc 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java @@ -34,7 +34,7 @@ public class CheckRocksdbCqWriteProgressCommand implements SubCommand { @Override public String commandName() { - return "checkRocksdbCqWriteProgressCommandCommand"; + return "checkRocksdbCqWriteProgress"; } @Override @@ -82,9 +82,9 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { String brokerAddr = brokerData.getBrokerAddrs().get(0L); CheckRocksdbCqWriteProgressResponseBody body = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic); if (StringUtils.isNotBlank(topic)) { - System.out.printf(body.getDiffResult()); + System.out.print(body.getDiffResult()); } else { - System.out.printf(brokerName + " | " + brokerAddr + " | " + body.getDiffResult()); + System.out.print(brokerName + " | " + brokerAddr + " | \n" + body.getDiffResult()); } } From df8757f09ebd868c3e512bd84d86db3c7707dbac Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Wed, 25 Sep 2024 20:56:33 +0800 Subject: [PATCH 167/265] [ISSUE #8731] Prepare to release Apache RocketMQ 5.3.1 (#8732) --- common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 8ac75a72c98..a03668e51ce 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ public class MQVersion { - public static final int CURRENT_VERSION = Version.V5_3_0.ordinal(); + public static final int CURRENT_VERSION = Version.V5_3_1.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; From b122c8109d62d7dbdacbb36f2b9c9efa49f9b859 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Wed, 25 Sep 2024 23:24:56 +0800 Subject: [PATCH 168/265] [maven-release-plugin] prepare release rocketmq-all-5.3.1 (#8749) --- acl/pom.xml | 2 +- auth/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- container/pom.xml | 2 +- controller/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- proxy/pom.xml | 2 +- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tieredstore/pom.xml | 2 +- tools/pom.xml | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index c9d5085dcc1..aad3831302e 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 rocketmq-acl rocketmq-acl ${project.version} diff --git a/auth/pom.xml b/auth/pom.xml index 71b07c33750..ac2fa6618fc 100644 --- a/auth/pom.xml +++ b/auth/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 rocketmq-auth rocketmq-auth ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index 7f74059a969..d2227f05104 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index 5a6c92f97dd..bdb69bf27f8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 82994c9a197..f7599d98661 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/container/pom.xml b/container/pom.xml index b9514defdb8..b85339d8d63 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/controller/pom.xml b/controller/pom.xml index 82b6fc7d969..86c4ac53f82 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 jar diff --git a/distribution/pom.xml b/distribution/pom.xml index 60fc6170bbe..04b81049411 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 7685a811690..9be076dce99 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/filter/pom.xml b/filter/pom.xml index 0acaa73f8ae..cf94c2ed0fa 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index d53540601e6..bd9235b4cc0 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 09ab5ed2586..d0ddf5687cf 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/pom.xml b/pom.xml index 8449bd6fb88..3bfaa985bc0 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -37,7 +37,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - HEAD + rocketmq-all-5.3.1 diff --git a/proxy/pom.xml b/proxy/pom.xml index 41e6fa95f55..86407092558 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/remoting/pom.xml b/remoting/pom.xml index 566c983ea98..e92b6e955a2 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 562a5ea2a33..ef5c33e7833 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 6de01626772..18bf24676fd 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index df380a0b604..c35bebe1d40 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index 96f042da21b..a3f9ef3af44 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index ee459dfd95a..69c1dc04342 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1-SNAPSHOT + 5.3.1 4.0.0 From daf3d1a666f3a9d1a9e314a124468e1d05f84ba2 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Thu, 26 Sep 2024 09:33:52 +0800 Subject: [PATCH 169/265] [maven-release-plugin] prepare for next development iteration (#8750) --- acl/pom.xml | 2 +- auth/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- container/pom.xml | 2 +- controller/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- proxy/pom.xml | 2 +- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tieredstore/pom.xml | 2 +- tools/pom.xml | 2 +- 20 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index aad3831302e..812dbd9fd13 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/auth/pom.xml b/auth/pom.xml index ac2fa6618fc..f7a5417860c 100644 --- a/auth/pom.xml +++ b/auth/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT rocketmq-auth rocketmq-auth ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index d2227f05104..f74c12989a1 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index bdb69bf27f8..e13d106a17d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index f7599d98661..b548d3df3c4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/container/pom.xml b/container/pom.xml index b85339d8d63..cc177abeea9 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -18,7 +18,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/controller/pom.xml b/controller/pom.xml index 86c4ac53f82..7092ca2b3cd 100644 --- a/controller/pom.xml +++ b/controller/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 jar diff --git a/distribution/pom.xml b/distribution/pom.xml index 04b81049411..88521fbede7 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 9be076dce99..19047c2f552 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/filter/pom.xml b/filter/pom.xml index cf94c2ed0fa..262177b61c2 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index bd9235b4cc0..012ebafe064 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index d0ddf5687cf..8ea4745b25d 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 3bfaa985bc0..ab4f9c45f67 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ 2012 org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -37,7 +37,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-5.3.1 + HEAD diff --git a/proxy/pom.xml b/proxy/pom.xml index 86407092558..e608d9f587f 100644 --- a/proxy/pom.xml +++ b/proxy/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/remoting/pom.xml b/remoting/pom.xml index e92b6e955a2..65e9a852fcc 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index ef5c33e7833..f6c5b3f54d6 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 18bf24676fd..d49de5ae267 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index c35bebe1d40..801a10301eb 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml index a3f9ef3af44..4d9af208187 100644 --- a/tieredstore/pom.xml +++ b/tieredstore/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 69c1dc04342..ab740bd8a70 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 5.3.1 + 5.3.2-SNAPSHOT 4.0.0 From b7a2a3dc3978279474dd183417515c6a22eddfcc Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Sun, 29 Sep 2024 09:54:15 +0800 Subject: [PATCH 170/265] [ISSUE #8769] Fix doc typo (#8770) --- .../org/apache/rocketmq/controller/impl/DLedgerController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java index be487849ce5..3421010340a 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java @@ -101,7 +101,7 @@ public class DLedgerController implements Controller { private final List brokerLifecycleListeners; - // Usr for checking whether the broker is alive + // use for checking whether the broker is alive private BrokerValidPredicate brokerAlivePredicate; // use for elect a master private ElectPolicy electPolicy; From 551c8c3d0da588ec8c358f3e1b5494031ca153d4 Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Sun, 29 Sep 2024 15:10:06 +0800 Subject: [PATCH 171/265] [ISSUE #8736] fix searchOffset corner case in rocksdb consume queue (#8737) * fix searchOffset for ConsumeQueue backed by RocksDB --- .../store/queue/RocksDBConsumeQueueTable.java | 33 ++++++++ .../queue/RocksDBConsumeQueueTableTest.java | 75 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java index c7d35fa8c0c..194bd4cca5f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java @@ -185,6 +185,39 @@ public long binarySearchInCQByTime(String topic, int queueId, long high, long lo long result = -1L; long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L; long ceiling = high, floor = low; + // Handle the following corner cases first: + // 1. store time of (high) < timestamp + ByteBuffer buffer = getCQInKV(topic, queueId, ceiling); + if (buffer != null) { + long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); + if (storeTime < timestamp) { + switch (boundaryType) { + case LOWER: + return ceiling + 1; + case UPPER: + return ceiling; + default: + log.warn("Unknown boundary type"); + break; + } + } + } + // 2. store time of (low) > timestamp + buffer = getCQInKV(topic, queueId, floor); + if (buffer != null) { + long storeTime = buffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); + if (storeTime > timestamp) { + switch (boundaryType) { + case LOWER: + return floor; + case UPPER: + return 0; + default: + log.warn("Unknown boundary type"); + break; + } + } + } while (high >= low) { long midOffset = low + ((high - low) >>> 1); ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset); diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java new file mode 100644 index 00000000000..d06b6da2fbd --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTableTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.junit.Test; +import org.mockito.stubbing.Answer; +import org.rocksdb.RocksDBException; + +import java.nio.ByteBuffer; + +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; + +public class RocksDBConsumeQueueTableTest { + + @Test + public void testBinarySearchInCQByTime() throws RocksDBException { + if (MixAll.isMac()) { + return; + } + ConsumeQueueRocksDBStorage rocksDBStorage = mock(ConsumeQueueRocksDBStorage.class); + DefaultMessageStore store = mock(DefaultMessageStore.class); + RocksDBConsumeQueueTable table = new RocksDBConsumeQueueTable(rocksDBStorage, store); + doAnswer((Answer) mock -> { + /* + * queueOffset timestamp + * 100 1000 + * 200 2000 + * 201 2010 + * 1000 10000 + */ + byte[] keyBytes = mock.getArgument(0); + ByteBuffer keyBuffer = ByteBuffer.wrap(keyBytes); + int len = keyBuffer.getInt(0); + long offset = keyBuffer.getLong(4 + 1 + len + 1 + 4 + 1); + long phyOffset = offset; + long timestamp = offset * 10; + final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE); + byteBuffer.putLong(phyOffset); + byteBuffer.putInt(1); + byteBuffer.putLong(0); + byteBuffer.putLong(timestamp); + return byteBuffer.array(); + }).when(rocksDBStorage).getCQ(any()); + assertEquals(1001, table.binarySearchInCQByTime("topic", 0, 1000, 100, 20000, 0, BoundaryType.LOWER)); + assertEquals(1000, table.binarySearchInCQByTime("topic", 0, 1000, 100, 20000, 0, BoundaryType.UPPER)); + assertEquals(100, table.binarySearchInCQByTime("topic", 0, 1000, 100, 1, 0, BoundaryType.LOWER)); + assertEquals(0, table.binarySearchInCQByTime("topic", 0, 1000, 100, 1, 0, BoundaryType.UPPER)); + assertEquals(201, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2001, 0, BoundaryType.LOWER)); + assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2001, 0, BoundaryType.UPPER)); + assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2000, 0, BoundaryType.LOWER)); + assertEquals(200, table.binarySearchInCQByTime("topic", 0, 1000, 100, 2000, 0, BoundaryType.UPPER)); + } +} \ No newline at end of file From 15641b6aeaa0fcaa55ea25aa0bb296d8d26cc750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Sat, 5 Oct 2024 10:07:00 +0800 Subject: [PATCH 172/265] [ISSUE #8733] Add a performance benchmark testing pipeline (#8734) * Test the performance benchmark pipeline execution status (#8759) * Add a benchmark workflow to current ci workflows * Fix bug: Use the correct branch for image generation * Update config * Update config * Replace controller image to fix the issue of the controller failing to start * Trigger ci * Trigger ci * Update test tool * Update test tool * Extend benchmark based on the original workflow * Test the performance benchmark pipeline execution status * Update test tool * Add a benchmark workflow to current ci workflows * Fix bug: Use the correct branch for image generation * Update config * Update config * Replace controller image to fix the issue of the controller failing to start * Trigger ci * Trigger ci * Update test tool * Update test tool * Extend benchmark based on the original workflow * Set parameter thresholds for the performance benchmark --- .github/workflows/push-ci.yml | 37 +++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index b23d69788cb..cc2a053eba3 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -78,7 +78,6 @@ jobs: name: versionlist path: rocketmq-docker/image-build-ci/versionlist/* - list-version: if: > github.repository == 'apache/rocketmq' && @@ -101,6 +100,7 @@ jobs: a=(`ls versionlist`) printf '%s\n' "${a[@]}" | jq -R . | jq -s . echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT + deploy: if: ${{ success() }} name: Deploy RocketMQ @@ -110,7 +110,9 @@ jobs: strategy: matrix: version: ${{ fromJSON(needs.list-version.outputs.version-json) }} + test-type: [e2e, benchmark] steps: + - run: echo "Running ${{ matrix.test-type }}... " - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 name: Deploy rocketmq with: @@ -134,6 +136,7 @@ jobs: image: repository: ${{env.DOCKER_REPO}} tag: ${{ matrix.version }} + test-e2e-grpc-java: if: ${{ success() }} name: Test E2E grpc java @@ -247,16 +250,46 @@ jobs: name: test-e2e-remoting-java-log.txt path: testlog.txt + benchmark-test: + if: ${{ success() }} + runs-on: ubuntu-latest + name: Performance benchmark test + needs: [ list-version, deploy ] + timeout-minutes: 60 + steps: + - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e + name: Performance benchmark + with: + action: "performance-benchmark" + ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" + job-id: 1 + # The time to run the test, 15 minutes + test-time: "900" + # Some thresholds set in advance + min-send-tps-threshold: "12000" + max-rt-ms-threshold: "500" + avg-rt-ms-threshold: "10" + max-2c-rt-ms-threshold: "100" + avg-2c-rt-ms-threshold: "10" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-report + path: benchmark/ + clean: if: always() name: Clean - needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java] + needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java, benchmark-test] runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: version: ${{ fromJSON(needs.list-version.outputs.version-json) }} + test-type: [ e2e, benchmark ] steps: + - run: echo "Cleaning ${{ matrix.test-type }}... " - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 name: clean with: From 6eb5a1019198bf69caf99ff3db6dc562162a398c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Oct 2024 10:08:54 +0800 Subject: [PATCH 173/265] Bump commons-io:commons-io from 2.7 to 2.14.0 (#8781) Bumps commons-io:commons-io from 2.7 to 2.14.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ab4f9c45f67..b18d9bbb439 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ 3.20.0-GA 4.2.2 3.12.0 - 2.7 + 2.14.0 32.0.1-jre 2.9.0 0.3.1-alpha From c5ac8a47ea9e84a05f63a26c771b25687d0e49dd Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Sat, 5 Oct 2024 10:09:07 +0800 Subject: [PATCH 174/265] [ISSUE #8782] Fix log typo (#8783) --- .../apache/rocketmq/broker/processor/PullMessageProcessor.java | 2 +- .../java/io/openmessaging/rocketmq/promise/DefaultPromise.java | 2 +- .../org/apache/rocketmq/store/AllocateMappedFileService.java | 2 +- .../main/java/org/apache/rocketmq/store/util/PerfCounter.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index d53454f215d..6dd8b300478 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -799,7 +799,7 @@ public void executeRequestWhenWakeup(final Channel channel, final RemotingComman } } } catch (RemotingCommandException e1) { - LOGGER.error("excuteRequestWhenWakeup run", e1); + LOGGER.error("executeRequestWhenWakeup run", e1); } }; this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request)); diff --git a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java index 36ac27f417a..46e607a5802 100644 --- a/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java +++ b/openmessaging/src/main/java/io/openmessaging/rocketmq/promise/DefaultPromise.java @@ -82,7 +82,7 @@ public V get(final long timeout) { try { lock.wait(waitTime); } catch (InterruptedException e) { - LOG.error("promise get value interrupted,excepiton:{}", e.getMessage()); + LOG.error("promise get value interrupted,exception:{}", e.getMessage()); } if (!isDoing()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java index 3dbc274ef00..d9cd602a65c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java @@ -132,7 +132,7 @@ public void shutdown() { super.shutdown(true); for (AllocateRequest req : this.requestTable.values()) { if (req.mappedFile != null) { - log.info("delete pre allocated maped file, {}", req.mappedFile.getFileName()); + log.info("delete pre allocated mapped file, {}", req.mappedFile.getFileName()); req.mappedFile.destroy(1000); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java b/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java index e2a55d63994..99649398a83 100644 --- a/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java +++ b/store/src/main/java/org/apache/rocketmq/store/util/PerfCounter.java @@ -356,7 +356,7 @@ public void run() { } } catch (Exception e) { - logger.error("{} get unknown errror", getServiceName(), e); + logger.error("{} get unknown error", getServiceName(), e); try { Thread.sleep(1000); } catch (Throwable ignored) { From 782da708d591f7de1211e9873896eec86457df72 Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Tue, 8 Oct 2024 17:42:23 +0800 Subject: [PATCH 175/265] [ISSUE #8786] Fix doc typo (#8787) --- .../apache/rocketmq/broker/controller/ReplicasManager.java | 2 +- .../java/org/apache/rocketmq/client/consumer/PopStatus.java | 2 +- .../org/apache/rocketmq/common/utils/ServiceProvider.java | 2 +- .../org/apache/rocketmq/remoting/protocol/ForbiddenType.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java index c294f860ba3..d12d142d6d7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java @@ -686,7 +686,7 @@ private void schedulingSyncBrokerMetadata() { } /** - * Scheduling sync controller medata. + * Scheduling sync controller metadata. */ private boolean schedulingSyncControllerMetadata() { // Get controller metadata first. diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java b/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java index 17dda9a2001..57fbe67bcca 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/PopStatus.java @@ -23,7 +23,7 @@ public enum PopStatus { FOUND, /** * No new message can be pull after polling time out - * delete after next realease + * delete after next release */ NO_NEW_MSG, /** diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java index 65dea47b5ea..49e2c442b23 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java @@ -50,7 +50,7 @@ public class ServiceProvider { * Returns a string that uniquely identifies the specified object, including its class. *

    * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() - * method, but works even when the specified object's class has overidden the toString method. + * method, but works even when the specified object's class has overridden the toString method. * * @param o may be null. * @return a string of form classname@hashcode, or "null" if param o is null. diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java index 0701dc57fc5..7c561f5721a 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ForbiddenType.java @@ -37,11 +37,11 @@ public interface ForbiddenType { */ int TOPIC_FORBIDDEN = 3; /** - * 4=forbidden by brocasting mode + * 4=forbidden by broadcasting mode */ int BROADCASTING_DISABLE_FORBIDDEN = 4; /** - * 5=forbidden for a substription(group with a topic) + * 5=forbidden for a subscription(group with a topic) */ int SUBSCRIPTION_FORBIDDEN = 5; From 483a48a130658e748d0640bd581fb9c925a5ddea Mon Sep 17 00:00:00 2001 From: rongtong Date: Wed, 9 Oct 2024 09:58:05 +0800 Subject: [PATCH 176/265] PrintMessageSubCommand support lmq (#8785) --- .../message/PrintMessageSubCommand.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java index bb82f5079e5..97e101d813c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java @@ -24,6 +24,7 @@ import org.apache.commons.cli.Options; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageExt; @@ -97,6 +98,12 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = + new Option("l", "lmqParentTopic", true, + "Lmq parent topic, lmq is used to find the route."); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -113,11 +120,20 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t String subExpression = !commandLine.hasOption('s') ? "*" : commandLine.getOptionValue('s').trim(); + String lmqParentTopic = + !commandLine.hasOption('l') ? null : commandLine.getOptionValue('l').trim(); + boolean printBody = !commandLine.hasOption('d') || Boolean.parseBoolean(commandLine.getOptionValue('d').trim()); consumer.start(); - Set mqs = consumer.fetchSubscribeMessageQueues(topic); + Set mqs; + if (lmqParentTopic != null) { + mqs = consumer.fetchSubscribeMessageQueues(lmqParentTopic); + mqs.forEach(mq -> mq.setTopic(topic)); + } else { + mqs = consumer.fetchSubscribeMessageQueues(topic); + } for (MessageQueue mq : mqs) { long minOffset = consumer.minOffset(mq); long maxOffset = consumer.maxOffset(mq); @@ -139,6 +155,7 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t READQ: for (long offset = minOffset; offset < maxOffset; ) { try { + fillBrokerAddrIfNotExist(consumer, mq, lmqParentTopic); PullResult pullResult = consumer.pull(mq, subExpression, offset, 32); offset = pullResult.getNextBeginOffset(); switch (pullResult.getPullStatus()) { @@ -167,4 +184,17 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t consumer.shutdown(); } } + + public void fillBrokerAddrIfNotExist(DefaultMQPullConsumer defaultMQPullConsumer, MessageQueue messageQueue, + String routeTopic) { + + FindBrokerResult findBrokerResult = defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory() + .findBrokerAddressInSubscribe(messageQueue.getBrokerName(), 0, false); + if (findBrokerResult == null) { + // use lmq parent topic to fill up broker addr table + defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getRebalanceImpl().getmQClientFactory() + .updateTopicRouteInfoFromNameServer(routeTopic); + } + + } } From a948f67b4dd711d74171e5d3559f6be7bd0c60e1 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 9 Oct 2024 09:58:52 +0800 Subject: [PATCH 177/265] [ISSUE #8796] Add more test coverage for PopLongPollingService (#8797) --- .../PopLongPollingServiceTest.java | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java new file mode 100644 index 00000000000..6527beeb682 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.longpolling; + +import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.store.MessageFilter; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PopLongPollingServiceTest { + + @Mock + private BrokerController brokerController; + + @Mock + private NettyRequestProcessor processor; + + @Mock + private ChannelHandlerContext ctx; + + @Mock + private ExecutorService pullMessageExecutor; + + private PopLongPollingService popLongPollingService; + + private final String defaultTopic = "defaultTopic"; + + @Before + public void init() { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setPopPollingMapSize(100); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + popLongPollingService = spy(new PopLongPollingService(brokerController, processor, true)); + } + + @Test + public void testNotifyMessageArrivingWithRetryTopic() { + int queueId = 0; + doNothing().when(popLongPollingService).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, null, 0L, null, null); + popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId); + verify(popLongPollingService, times(1)).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, null, 0L, null, null); + } + + @Test + public void testNotifyMessageArriving() { + int queueId = 0; + Long tagsCode = 123L; + long msgStoreTime = System.currentTimeMillis(); + byte[] filterBitMap = new byte[]{0x01}; + Map properties = new ConcurrentHashMap<>(); + doNothing().when(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + verify(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + } + + @Test + public void testNotifyMessageArrivingValidRequest() throws Exception { + String cid = "CID_1"; + int queueId = 0; + ConcurrentHashMap> topicCidMap = new ConcurrentHashMap<>(); + ConcurrentHashMap cids = new ConcurrentHashMap<>(); + cids.put(cid, (byte) 1); + topicCidMap.put(defaultTopic, cids); + popLongPollingService = new PopLongPollingService(brokerController, processor, true); + ConcurrentLinkedHashMap> pollingMap = + new ConcurrentLinkedHashMap.Builder>().maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build(); + Channel channel = mock(Channel.class); + when(channel.isActive()).thenReturn(true); + PopRequest popRequest = mock(PopRequest.class); + MessageFilter messageFilter = mock(MessageFilter.class); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + when(popRequest.getMessageFilter()).thenReturn(messageFilter); + when(popRequest.getSubscriptionData()).thenReturn(subscriptionData); + when(popRequest.getChannel()).thenReturn(channel); + String pollingKey = KeyBuilder.buildPollingKey(defaultTopic, cid, queueId); + ConcurrentSkipListSet popRequests = mock(ConcurrentSkipListSet.class); + when(popRequests.pollLast()).thenReturn(popRequest); + pollingMap.put(pollingKey, popRequests); + FieldUtils.writeDeclaredField(popLongPollingService, "topicCidMap", topicCidMap, true); + FieldUtils.writeDeclaredField(popLongPollingService, "pollingMap", pollingMap, true); + boolean actual = popLongPollingService.notifyMessageArriving(defaultTopic, queueId, cid, null, 0, null, null); + assertFalse(actual); + } + + @Test + public void testWakeUpNullRequest() { + assertFalse(popLongPollingService.wakeUp(null)); + } + + @Test + public void testWakeUpIncompleteRequest() { + PopRequest request = mock(PopRequest.class); + when(request.complete()).thenReturn(false); + assertFalse(popLongPollingService.wakeUp(request)); + } + + @Test + public void testWakeUpInactiveChannel() { + PopRequest request = mock(PopRequest.class); + when(request.complete()).thenReturn(true); + when(request.getCtx()).thenReturn(ctx); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + when(channel.isActive()).thenReturn(true); + when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor); + assertTrue(popLongPollingService.wakeUp(request)); + } + + @Test + public void testWakeUpValidRequestWithException() throws Exception { + PopRequest request = mock(PopRequest.class); + when(request.complete()).thenReturn(true); + when(request.getCtx()).thenReturn(ctx); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + when(request.getChannel()).thenReturn(channel); + when(channel.isActive()).thenReturn(true); + when(brokerController.getPullMessageExecutor()).thenReturn(pullMessageExecutor); + when(processor.processRequest(any(), any())).thenThrow(new RuntimeException("Test Exception")); + assertTrue(popLongPollingService.wakeUp(request)); + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + verify(pullMessageExecutor).submit(captor.capture()); + captor.getValue().run(); + verify(processor).processRequest(any(), any()); + } + + @Test + public void testPollingNotPolling() { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand remotingCommand = mock(RemotingCommand.class); + PollingHeader requestHeader = mock(PollingHeader.class); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + MessageFilter messageFilter = mock(MessageFilter.class); + when(requestHeader.getPollTime()).thenReturn(0L); + PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter); + assertEquals(PollingResult.NOT_POLLING, result); + } + + @Test + public void testPollingServicePollingTimeout() throws IllegalAccessException { + String cid = "CID_1"; + popLongPollingService = new PopLongPollingService(brokerController, processor, true); + popLongPollingService.shutdown(); + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand remotingCommand = mock(RemotingCommand.class); + PollingHeader requestHeader = mock(PollingHeader.class); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + MessageFilter messageFilter = mock(MessageFilter.class); + when(requestHeader.getPollTime()).thenReturn(1000L); + when(requestHeader.getTopic()).thenReturn(defaultTopic); + when(requestHeader.getConsumerGroup()).thenReturn("defaultGroup"); + ConcurrentHashMap> topicCidMap = new ConcurrentHashMap<>(); + ConcurrentHashMap cids = new ConcurrentHashMap<>(); + cids.put(cid, (byte) 1); + topicCidMap.put(defaultTopic, cids); + FieldUtils.writeDeclaredField(popLongPollingService, "topicCidMap", topicCidMap, true); + PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter); + assertEquals(PollingResult.POLLING_TIMEOUT, result); + } + + @Test + public void testPollingPollingSuc() { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + RemotingCommand remotingCommand = mock(RemotingCommand.class); + PollingHeader requestHeader = mock(PollingHeader.class); + SubscriptionData subscriptionData = mock(SubscriptionData.class); + MessageFilter messageFilter = mock(MessageFilter.class); + when(requestHeader.getPollTime()).thenReturn(1000L); + when(requestHeader.getBornTime()).thenReturn(System.currentTimeMillis()); + when(requestHeader.getTopic()).thenReturn("topic"); + when(requestHeader.getConsumerGroup()).thenReturn("cid"); + when(requestHeader.getQueueId()).thenReturn(0); + PollingResult result = popLongPollingService.polling(ctx, remotingCommand, requestHeader, subscriptionData, messageFilter); + assertEquals(PollingResult.POLLING_SUC, result); + } +} From 95b1c51f86592cbdb04b3827f95d571e883af804 Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Wed, 9 Oct 2024 10:37:19 +0800 Subject: [PATCH 178/265] Add Utils for put header to Metadata to avoid duplicate data. (#8792) --- .../proxy/common/utils/GrpcUtils.java | 45 +++++++++++++++++++ .../AuthenticationInterceptor.java | 7 +-- .../grpc/interceptor/HeaderInterceptor.java | 9 ++-- .../grpc/pipeline/AuthenticationPipeline.java | 3 +- 4 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java new file mode 100644 index 00000000000..5c50de4426e --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/GrpcUtils.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common.utils; + +import io.grpc.Attributes; +import io.grpc.Metadata; +import io.grpc.ServerCall; + +public class GrpcUtils { + + private GrpcUtils() { + } + + public static void putHeaderIfNotExist(Metadata headers, Metadata.Key key, T value) { + if (headers == null) { + return; + } + if (!headers.containsKey(key) && value != null) { + headers.put(key, value); + } + } + + public static T getAttribute(ServerCall call, Attributes.Key key) { + Attributes attributes = call.getAttributes(); + if (attributes == null) { + return null; + } + return attributes.get(key); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java index 28ee019fae7..e082ba6e28c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java @@ -33,6 +33,7 @@ import org.apache.rocketmq.acl.common.AuthenticationHeader; import org.apache.rocketmq.acl.plain.PlainAccessResource; import org.apache.rocketmq.common.constant.GrpcConstants; +import org.apache.rocketmq.proxy.common.utils.GrpcUtils; import org.apache.rocketmq.proxy.config.ConfigurationManager; public class AuthenticationInterceptor implements ServerInterceptor { @@ -49,8 +50,8 @@ public ServerCall.Listener interceptCall(ServerCall call, Metada @Override public void onMessage(R message) { GeneratedMessageV3 messageV3 = (GeneratedMessageV3) message; - headers.put(GrpcConstants.RPC_NAME, messageV3.getDescriptorForType().getFullName()); - headers.put(GrpcConstants.SIMPLE_RPC_NAME, messageV3.getDescriptorForType().getName()); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.RPC_NAME, messageV3.getDescriptorForType().getFullName()); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.SIMPLE_RPC_NAME, messageV3.getDescriptorForType().getName()); if (ConfigurationManager.getProxyConfig().isEnableACL()) { try { AuthenticationHeader authenticationHeader = AuthenticationHeader.builder() @@ -85,7 +86,7 @@ protected void validate(AuthenticationHeader authenticationHeader, Metadata head if (accessResource instanceof PlainAccessResource) { PlainAccessResource plainAccessResource = (PlainAccessResource) accessResource; - headers.put(GrpcConstants.AUTHORIZATION_AK, plainAccessResource.getAccessKey()); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.AUTHORIZATION_AK, plainAccessResource.getAccessKey()); } } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java index 1de2ce4f986..e3e78841559 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.constant.HAProxyConstants; import org.apache.rocketmq.common.constant.GrpcConstants; +import org.apache.rocketmq.proxy.common.utils.GrpcUtils; import org.apache.rocketmq.proxy.grpc.constant.AttributeKeys; import java.net.InetSocketAddress; @@ -44,11 +45,11 @@ public ServerCall.Listener interceptCall( SocketAddress remoteSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); remoteAddress = parseSocketAddress(remoteSocketAddress); } - headers.put(GrpcConstants.REMOTE_ADDRESS, remoteAddress); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.REMOTE_ADDRESS, remoteAddress); SocketAddress localSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_LOCAL_ADDR); String localAddress = parseSocketAddress(localSocketAddress); - headers.put(GrpcConstants.LOCAL_ADDRESS, localAddress); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.LOCAL_ADDRESS, localAddress); for (Attributes.Key key : call.getAttributes().keys()) { if (!StringUtils.startsWith(key.toString(), HAProxyConstants.PROXY_PROTOCOL_PREFIX)) { @@ -57,12 +58,12 @@ public ServerCall.Listener interceptCall( Metadata.Key headerKey = Metadata.Key.of(key.toString(), Metadata.ASCII_STRING_MARSHALLER); String headerValue = String.valueOf(call.getAttributes().get(key)); - headers.put(headerKey, headerValue); + GrpcUtils.putHeaderIfNotExist(headers, headerKey, headerValue); } String channelId = call.getAttributes().get(AttributeKeys.CHANNEL_ID); if (StringUtils.isNotBlank(channelId)) { - headers.put(GrpcConstants.CHANNEL_ID, channelId); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.CHANNEL_ID, channelId); } return next.startCall(call, headers); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java index 58eed91c9fa..e317b48f1ed 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/pipeline/AuthenticationPipeline.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.utils.GrpcUtils; import org.apache.rocketmq.proxy.processor.MessagingProcessor; public class AuthenticationPipeline implements RequestPipeline { @@ -73,7 +74,7 @@ protected AuthenticationContext newContext(ProxyContext context, Metadata header if (result instanceof DefaultAuthenticationContext) { DefaultAuthenticationContext defaultAuthenticationContext = (DefaultAuthenticationContext) result; if (StringUtils.isNotBlank(defaultAuthenticationContext.getUsername())) { - headers.put(GrpcConstants.AUTHORIZATION_AK, defaultAuthenticationContext.getUsername()); + GrpcUtils.putHeaderIfNotExist(headers, GrpcConstants.AUTHORIZATION_AK, defaultAuthenticationContext.getUsername()); } } return result; From 6c90aace3659de705abdfda16761c338853291af Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Thu, 10 Oct 2024 09:48:30 +0800 Subject: [PATCH 179/265] [ISSUE #8802] Update controller design.md (#8803) --- docs/cn/controller/design.md | 4 ++-- .../image/controller/controller_design_3.png | Bin 77603 -> 70160 bytes docs/en/controller/design.md | 4 ++-- .../image/controller/controller_design_3.png | Bin 77603 -> 70160 bytes 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cn/controller/design.md b/docs/cn/controller/design.md index 563a624eddc..13eba7764a6 100644 --- a/docs/cn/controller/design.md +++ b/docs/cn/controller/design.md @@ -121,13 +121,13 @@ nextTransferFromWhere + size > currentTransferEpochEndOffset，则将 selectMapp ![示意图](../image/controller/controller_design_3.png) -`current state(4byte) + Two flags(4byte) + slaveAddressLength(4byte) + slaveAddress(50byte)` +`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)` - Current state 代表当前的 HAConnectionState，也即 HANDSHAKE。 - Two flags 是两个状态标志位，其中，isSyncFromLastFile 代表是否要从 Master 的最后一个文件开始复制，isAsyncLearner 代表该 Slave 是否是异步复制，并以 Learner 的形式接入 Master。 -- slaveAddressLength 与 slaveAddress 代表了该 Slave 的地址，用于后续加入 SyncStateSet 。 +- slaveBrokerId 代表了该 Slave 的 brokerId，用于后续加入 SyncStateSet 。 2.AutoSwitchHaConnection (Master) 会向 Slave 回送 HandShake 包，如下： diff --git a/docs/cn/image/controller/controller_design_3.png b/docs/cn/image/controller/controller_design_3.png index 8c475bcecf1bc030cb9983c01ff07c9ed1433fb2..0379c231d46ed8ca7e6ab3a9c094b78fdfa683d8 100644 GIT binary patch literal 70160 zcmZ^~2{@GR_dh-)5<-$KMuehJLWm&>A{tj z8H~X&X3WfgdjI~v-}U+Q{(i6P@yuM0`q_q7yI)^r~ByA4(%c5i@O#tj3Lf1{O!D*0LGrr zyzG2lI9c-y(B2RR=xS-01bp0_^S8awIR)CmR{1N>Z9HhUta0jXR;*v}oUJShXSfsn z=8kT#@%)>nz`fPOciC_5pVbMAR6qOpvh;7yB~J(4ZPS`3W5E#8F(qxPSyfk|?JU%H zvA)B9c)QOl{R*usP9bg#(JBry(=syJO?Pp%7Bt;C*D{bT;erO@U87a0nMn5`m@Y{N>|etBF3d28Fn{%?a}yr@&U zT}0qZ#6_7+{VzA7&Ml=$o!l7xUnp%MWE~{FdH3^e%10UZl-`RfoX2?vXRdw7D(d6` zhv5FFo5~tj)y5^Yoi<7NhzR&hcf;gwo^w-HFO@p}w>J=ziZ8s+6cf9lPHWKo8+Te@ zF{Kius@vUk|0~V@)!GTpYGOw4Q^FnGm>?VjFkGH820BGiWxqcJ|!8h zOcZl$B+9=?CJV@MJtSRS^#1`0%6S2^;Z*<7-y;#KlWrP+(K(rogDWjIhWkz)vdkt~ zUzS(=r;*QmM?>tNowdgDn2eq?9ev%e!!|%Hkt9{fTB39HQtPGruA~n^7GTEmfH&`jxY?_B%v{3qNo9}Ir+NTYFX?DJN9u(dXVE<#yM6zk zH2}fL>P|<`G705?8b=Fwhcpbveu&}vPvJoJw9J@w2iZ5?EOc~nZ?rtA!78FHdy%gL zOAlAq(&9T2%)Xw?8vHqvuDu~uIxRb;74T_1<<0$jVMT|Jr5>=3x$D#_^#73t>(#Xj zO2&^&E|SedaP31N-<;(a+jmpse!f^C16r)LNyy55{$}tDtTk?H92~>fW*e{im=(+4tDi5a)=?A+i_i z`ohPyf8RM(A;+$9E$pd8EQ%TB%G75&%;FQPAv?voDlEeBQ#Ljx?I(X*_*TswGY7ho z3jBHBh^!Pdnu=lV!9)aMF}j4kI<jREawXoeF*;(58(0V5Px=}kyNzOFKB+~R{-MPiI1KCL0VRamTd0KX zH2k;3X0Hgia`~n`v932(iQlp>?LY7g4&mVcM#$P9d#B&*td%$z!}_Q{ywz4j+(PrF zGoIZD$-!AQ|LI<^R>eJM)@q$sd<-Clr%B)ufZ$vrlo{^#8qa^LAjBb5P1flR>zAU z5YZ4v#Knsu@;weUkJ;n^3g@~U9ZDh9Tr?GKoKL0D%-F)ZmH8t_$y-kl!2dLz$Y?^8 zDZ|eJ*C}NnX3HJo)VUsZ$06VL>BR&>)-H3O>yvzBDnOV%tyOrqb3KwloozjBQt0!u zcj=pLnFEj`G}E6g%H0`@+DW8M1}bl{leltqfXyXd-C79GPujueDNl78guzc>Q4agthDNpQwWG=pn)xJ6z0^Lk%Ez5J(bplIrp0jf&&EwN>HMQBMF^72R8`LF zLpMxWJ?Y=11=qaan!On45iYOQwkXIhuOlIaz)J~3QwitxW7E_-Y#XgPi1b7jWz}K?a}|gZ|ZYRn~ z37NG1Z|)@RCfaU)axSw6D4oRwSML`ppgS(GzSN}8wdMaaRdu?p@;wVz=}cbRFNQ25 zaO&MC{`@Nmt+rn>mwz6VwF%J$O%~L~{c~*oqcc1EIYgPaz8c(~e`!tC@o&X6V=t*{ zDJIrOe6hDI%0-1sniO801k%oE#aLE*X9ZSc1c%*v@Vd#=xvMW*3w%p1aJI5CPRY$_ zDQ8N?YM@!TV%lV58aoh@QfhCT8!KB5M0Y`GLMQSx)YlT%{ZTcsTD}G*;&uo}k|$Tq zZBE4#K#}YbtdS3?TX|rw?fs&NslGBhej0~WLZdVooX?D4{^-m9<15y#CDT;ok(Ff% zBjeAv3EkHX?w`>pGZC&amzgZ#ePMYCIza$LoRYPKi33X7dl_z|8HB}*p5?MmlQdHV zSPOHF63@!-o`}$J{d}%8+LvK*fFzYoS&1#Ro|uQCHVxMb)^AXiqueY8{yY(lp(t+!^ zVK8;EXsO?TEhvX8lR?vi>B~fGn-C%?#{4$9*TGU{EfgI-1#i$k@=aOcd2m5jXG^oL zxa&~#khxFxvO5vt5|Z4%*0Q&^jJg^wrgkt+l@YyZTfF1&0cBbZgs7BUrKgNp|{AdkQRP=B8>a9U?%j< z9uv*x%FTe@seKAUXV}u)s+!u*oiBusM&MIaORYWRK?&O}%6E+QSdv*!|o2aXz` zk>xc+85HR>FDlsS+V6(b_^P8ZWD2*zp*ObrWnop})N}y#Esi;Hd^2xmAxW(}=b#+pE>>en> zDi%uLx`YqiGDO@93blI(n)!hI>W|-5s-06ke9aWRgChqJ(ep3t8Bmz^^}o*Y4_H6P zQ|H^;9&NtFvlx?J3JzJu^NiYxT)Kf%Stv<=nlea#suX5FtUb&BmUA*v-uHV&2IC1^ z`P)_JZu{l9ch`9TX2`Jj`l(O3fHL&dN1yOB0d{@HWE=3t+oag46`DFw) ze`xdKz{J=%*>u=64%s}|`sq4XF4wzLLBCnW>9tSVH#{WEakVq(ztJ$yg+5r_RQBK8 z={Tv!b-BYl%l`LdUs|M=nX2X>?;DV^89pMPJv4A50?eA7!|R7Z5@~`20d+X~Zh0k8 zHII_ij!L`rqnN>j`xm`o!4LP zbDw|oa%F|^caN`=Jk;fY)7_u%ofqu7MBygUju%-@Chfis@#Ji0W1lQn>p1d#*I_n* zMN02MUmUZDZ0Uj!E3dwIIIs1xKsoygaJHthyx99!T3>JrNshN+j42a4mINA=C`hjj z32IH#Z_P*^QETYHW)L*`964h6WXtc7D?H(h{=Wz7RtCZy#Mf&E8VtB%V)6HGj26XK zv-swI*%mj)5uo4HBJ@SfE%2y;BL+?K0&rYLswrUgXUk;d`%ED+5llrXRw z9%!$KF{<|k2_@z*c>&(qsM@bw3d1i*IJ-S#uA0~Ab3bx__gHD18#+!*aXAC*`-se= z{(N2oHa2T%_s(u`S%USQzz#Z~atolwqd4{cb`7x_As0DtSYa*uVr-tq7cJ6U>x6EqyN_l(~6C zVgNi18?8IQ!RO(=wix2q#p8^x!#ZFSef!;Cu_JP8GDB*k8JhT1|0kf}3B+xmW!u$i zy+=%}Q8eu%+@$A`D6*`e-=UkDp9tEIdTo59wP z``m+03_Ux+DARPvB3(-&k3rxd=$EV1+hK<`ume4I zwLj6a)0GhyclR*#A_o_QbX52n8+6!DGfL=UNzu@jv<{nO9&3=-Y@Pz8C9$OKw%@=@ zO70B@uH-?Xd2^c=*Ds2d865Y8xF6P>uo^c#$D=SladaG>r?w#nX;!&jHmEX4F3+h6 zA}0Fh9SlC@qINbVH<~qNd{ndj(O)ZPd_WTxcB}vh_Uz96>4OzSwhFYazC~I=tZeX& zot$Tr=y!~VF^CN>uVUV0mgZmlxvtgySz_Gl2M-3*pd*6MuQZP`-SFP&^X(=+leelG zFZ-$T4f**NP`q#-I2!!xS1ikiw+pUdDvX4JjGMeyRFuC*JbtcQ^eiI!fwgy4*P1`k z&4%D$?bP`_7-oXsk?Kh9lRIQ2xyK7w&EC^$-zw`PhF2@V);2Z~CMq7?^(mLOUUaVobLAtZatI z4~Q3@Xw%Ct-msO`m)&qE@w;Vjv@!K=te8vQ;AAT;4wk)VJC~iNVT-S{&DtLnRB56% z%kznL?E~d)j>LEdC>vHuH|0kg=dwFmLanSG4+23Q?hh=3;&)?Nzv^xLJnZV^fp_-~ zs$9S6Gad8HXSxHq0+!*1_hvrt-RgVlUW`HpTEDns?_;ca5M^o`(9}{uB(l_NvwK?l zkL02*D7sW`RZS6zGBB^FUH3TnWIx;6m&C6=h{aT4L{vzFZ|tl(TTcbE)^V2r^nB4u1uMFEL)?i8dRX(8w$ z8tq8fpX`N{8C~knfimv9M?Px42#~nVky54^@@UjBW&zK%XiOTN7(bUam9F5m(~3*Z zQk$Huea6O>1aIt+eY&3zIFpL}SoIuV?SK21I3>l9YLAUFmE}x+e<8+z<}=m53BKt= z!Rs~?`5l+tZ3H&r8@A=Zu3WH#25yoR*(-GsX3(ci&4 zPME^4dF%N=XbHhxi1TyuJL~e8kBKE;6`OpXl71M5E|DHm(iHG8wC#fzPwwQ&6gM>8 zr0&OxHr8%Fp-j8Lh|z@Qn=_z>V7EZKwqLp<^ThVS3^b-&wv=LpgzQbH<>h_&@uL(k zrM`?KWbkA5424igCvtxh)MxhNT|cty9?b)pADXjj+quW|dFgD@_EgZ*u7YN_R1^*l zT2HyI(}^1WRRP<}SdYvbr9{j_i9Ao8+CA!?Z11?j%@<3!xNcqeI|M#@U8K+A688JD zu9JO=HIIhJ)tYG5U&*B&OB^&*hfd#-t!STK20)Y${}QN>b;j1>;YR0`JZ(<~Spi{H zr7rYHo6ca(m-@U1BxR+OWV{TysHoC&Rbo5)u!z*XLLc5gUOarZ+9rxwH*~mdveEA- z#%=Y|K6)+(Uhg#(2O_(oR^@XG`k(&*zwvJ%%x-)O*L+7hV}#4>7~{@oswr)xv$(_355Vg&3E zVdzu_bLj-aLkK)DMm{t0iBDO*VB~7ljPsngi^&UjO^<#nL-PL5Y>z?m1P@y(sYV9o zLXh`);%VNtF$<$CtK^?t^Dhf>7{3Ou!NBP%p1lg`UYhYI{KOuU1s^Hqo>+Tu0eU2X zQuW`r9r)odan+&#E$+4z_uZ$En8Xh|tc9Ihz!fWBmg!+?19*n)3-qc{$9O1fn4@}a zEMTxcz)m9Ep)$J*^TNg(QndX&4A7a-8Q8JED^df{ zH~zr@(NeL3N0tSwb`Mf^4(BS!wK&r*b0C=iJh`???G*ulfKm=^Lx?yad-|=rS-^}8 z0e%d6hERC0W^#@PFL>idjz3s7DYMF}YX-52`sTd7b*Msq*_2FkJ$CP%6gnF78N%sm z0dfn9Z+~BM_#Syu&%ZomtFG0L z@M~@e>ZZB&pbf`HVlkr3W^V$tIYZbc!fva04%jzRs>ZK@8eH`3u@v}m67|6rh=ZXj zKX9)^#yyaXm7)-qU7*MBO{uI#KAPkV9?C6WT)%PZ9Dizt^P9MfA=7Koc)Wn(jB&<- z-(T)}k7BFYBkvaFOJCv4#kW#kUON?bQqETU+*RFU>78`-pkH@|?aAr%757|OMPy?% z8s^?8|HwOS*Gu=w99bEcQF_XhWuY9`CRx43ldZ(2P+$#{s<-ZuXy>MF9VwlqJjvj} zH5thOwA3MTKNqqEJxttPJXWBjS?-B8&nGK2x|K+N!~rK8velQ?o)j)IB{M4oPHBR( z>Ig&~@VV1$c&^;cs9~t7 zuWz6(?J}U*Q|z9$@rZsBXe)x7DH)<)YvQy<@WnGM(Ef+aw0dx z&}1|bP7Shm#eVj2IpaCQTM7eooal0tu0LG(l6EVsDZF*m2`>C8zdjImA}KLL3AN%E z@r(8YoFi+k9j~~wKjIC;ZM-Sub<_>3; z-8#P-18h6!ZY@ZKGc6Qox1>))-t??{H?E>^>USQaSh|w9vq5u$GgPtTiVbAe-k4U- zmU&9b=!BU>p$lvJt*HZ1k*1+aoX7FUL~K~V@$B6m!wp4QF$debx_0y7?uFn01)kQl zslD8##BEF6zXileRQrByc_(i9MIrxd^lp=?V zYT;qD(?$hJ&Yjo1#~;OcyG4BRW-4^Mgo_8oyAe-cDZ|mcOwnddZEX0KS6+N)j6FiJ z`PjVT`Ccwcr=R!BqT(#)#5^vY@a-FmMAG|@svct?Ksf@DUS~V?ZZPD!T6JxMG&ABL zP)v<;@>l(L>|Zh7dmzy=J>j=|5tJ>k@*sTcY}QnkPrh*0<6~(YepsP-|5cfIj+V7@ zRn9~bVEpDASFtw3W049Qac zLkj-zctk?G|D*OnhO}+RzLKkOIa*^%xJT6gh!wKL!mQsAg$+40|m-G2DFx3s@t~kK~KlPg57_a$BgR5W`zXg;6%g9Y+ z$U$q!8N0(7J#p$G_QZ;Z&QFgwG__M!!IxEG3i*(Ghi+7l=_FSLyHS|R3RjE;R=5?z z4){`1O7pe$2*;_r0z^S%=t*5#yZR(6HM0sgN9Q0u2{jS^z{~yWWNI92 z@uk4P1*JDt8*_HX&eIf18HSEu(S_3rQYOorpc@Z)RI4)swSNq@A2m0dqTzSCvWZ#p z@Xh4MB;rE}C`QwMX2NhN-59`_2tUiYAa-UPrhn%V?K0KX)$~nKM^z))@4(2==T1`Zs)o8d`Ux?Yd%*#q&srudFi>Z`2 z=XH$yUF*eUArVG$r4NtwE#G|_0*7Ab^I=Lzzjeu{ScYTnM^apDBW@3DMf{!mK(LR7 z*e+>}Z!h`;LdX=Cu7m9+@}frf;s31#5I~tzI!^X}oj^?22KJ9S)fN*5fhrc3Ph{PD z64dn*Y~tGizYny~re3cJq9P(=!-CwfyU#D4piBm&&V(Kj>E2Ashz_q!?7bBZGbZk{ z6NMSiJ_{9UdCjn%u{jb}^5-E(U>*>Memk+_lY)1ZW)U6O%+eky|R_R~|HSDRRR>B2?O6R5-20V?2nc!DGsv68 zm-R8^cJRaqiZHv=Eyw8cU`y_4S`{oYLyeTR1`8H(^VjsPR^$-=yjoD;5L`oWSd4PE zIiUjOV$dQUdo22E2TX-^7o3iWm5gUezomSA)$k-HYZE9{jH#_>Xqk3#?ew}wl=33nwe=o49<#UJ7talk`0f2mClmay5KDMSS zl~+Ad;%pR4PS?a1?u+M_MTF!P1`a_UbIT`2U}I}@YfE!uYp}H?*xLGq0=v40N44Y! zb%tF=`zAW%_V~i28sZD}@yOo)5Yb!JUkFwSm<%q}eHdB|Nd%1EReP?p?)pggxO|7% zF;tvCJSVACl|nzIr#-1Qe&MR#iV6Gp44X)@sh?RDY`nO&e8-Z-c9`;BtX4}apr(a< zf1p5KNnoPYDdFP{A6zJV*QLU5IBTK9H+|hS{x2{QT5IqS5WdQxvG!x%w7lc@bAV9B zBwFxui@M*v=u?a`u;tK<7X z^-@U0rI+$<3|}7$*;>GSW|pjsA5wjs0YC#x+5`m>sk}=@py^T=p znltu?4^mIymLrawga=+1w*WD$`w0z0)6!x-s1Z67t}m6{<6<4+1F8ERSGdjWmh{BY z^?*VbC)iOZRY{+W`52ErfBtO0RQk2?Aqj$lo?4(8+6}Jj>1e}o$~AD%Z8foLE+dUJ zIH|>F1tWxj>@Es8!*JWH(_o+_d2wyxz7b4#mx>WVgL2C}pEk;Xz*RROIrq#}52x zXm~zx&98pekD71m###pV!bQgZWCz5p9~S|1<(lb zxLs}1F?6LWwl@bIez;wbxFp7)T&mE6z0Zv}+soPs3A#PsUl$Ca9m^MRzz zA)T3JK`?pa`QH zTZ7Dw$@daMf5(C+Ow3kSHiW_&7s|$RcnS9E`C~GOdn=(@pbXh zq2WO;(slE%K?>K8Y7k_99|z6(@XaO5MsJ03C@Bf)71*@oJZOxg4T5&{O!z32r#BJA zot5i6PK4nmr2WQ0Rvv{OWZ4(J)r*J3T)LYIH-F`0qRU2Z(+Q3z4FXU6_>D8x?r&1) z0SfZP92}jPzYKFU`PPLI)4V3>=sZpQp+^IY4A3(Wnt~P_2>iy~EqdoF*#kRV34YJJ zuM3Bb%7-rz{JfwQNr>$N)-cgrdg0f42ASm8Mp_)y@?~8I2^!|vVWgkG40_vA^`z`8 z6UH=WVMYzn>VmPpb^9;OMf+%6G0iH5Ygi!GuG7wGTy0;^i+3SrD(()=XwsPpq@8m9 zB_ZEiP&j4bG-4H@5<7J>=6$=F^OuXjN&Ypu-{t#Tmf=RAN^@jhKDpp(^GZFHxLfZK zy}S2dU-0(s4d|W6%0Ek`WyE`dD#2~jJn(ty{PdB(>IZ>KFL$r;0Zzlgl=@{3==s-c29j$4qHNZ|5gf z6-!A$3lX1(@UBO$H=;CHwaGJ>*UHK*1_lO&hih-_W{E43bg@ zt6i>}t`(?-6&ePIa1B2RkE%`fW%WYG1RIq^GOo z&x$uqNi71>NBpv*nrlgdtV6Ta-H**1=T&2HNTv?6zm@sfD27x6#{izQu_@I zWObQ5kBC%9bYkiT_yA|>F|54SAy#r)}G65t!=b@$XSz^qqw2FJ`-o-y>O)FGN-2Rl%a zI6VWFB47zPnmE2Q^AO}Ze&ZWuOHQi`54K4gbPo!loI(t4R80j1be-p@d|tnlISmm$ zQJvycgN;pK_%%E`t3kFJ2}o6GX(+M4VW_KjYJjO9??*j+XkuaZ9+k4Sm+MLOAbVkG z(KH*WZ0#(R2FVGlRn}Kl7Uc-BD)fKscS{TC`JDNVcwn$OCsS!i&6_8mB^~H(&w;8@ zVRp$bvZBFDXuVGOFa!`b-Q(?Y%*FG>P)jLu{K8aoS$%!rY*VMlB9m>a5m%uHnki49 zV0Xzl_{2|@w9s^78`GHKll%N!yJxl{p4qfedUs`BUgk*sRX|~fphhXjpImU#Q^>m9 zC6=+N_rT2eJzCtIC%NyQ1ilDKH!x#U3i-p;Ep|3$Gs*Ahzv9>rkWbcUy^jYNS{vA8 z(uV2pgsS9L2i>f@txrpkjCW0YwLnA5GRdZmrP>%91Q8_kivH2v$|;0tAB zW5$>1wjAI}f^S^|ecwu##VP}6&bE(3Xjv6>@@51s0^fQtjMp5ub2A2Ut%BEH$J}ZN zwwlkmP)K8UxPxV*NK;E?)%jbP==7}!A@>vL^FC*aV2{9b4=&;%iS;UKEzi5J0Qk|x zgF;HSq`tBe%JEFDO6?cX!1SfeYCG=h?}mrB4rtm0$Zh%^lDSke0}3@FX~Y0G;zjP= zeHgw)3y{U11Pj*ruE3$D=GI%K)bS6i-{lG$+(-~i#P<@aUu=@zk;@sZR4?*(QhZWE z9Q~Wjs>Xt^SCv9-r^DI=iL|)xhC0){m0L!c;177UCwExnx0V4yG+3@zAm)!{JdtTG z5Ov*o{JR29ZUfP^Aq({;EUnr6p|!@ptGjg0108K(JjR}!v(%tPc1#e0j~0S0DM!B4 zka72Cem}rqV`Ceyp>lZLdlT8$TXWl`RI-&~6X_V6mOk?l7=y#&5j9;C4c%(rmj*r} zM{fqMj2Moankg?qo_R>$PR87V-9bFuWTyEw2VhN^QMTHD7s`}{>i?4Z{6e|$D*Z-6 zP(#$mhea>DCF4Hvk=YmhUk$Q;(Rgp8O~3Bp9%f-+{cKRULLT1Xo^mp5Kg@t_MMt}E z+p4C^%MmvHxG?ll{JGfho_C8pEmXV>UxfX;N3+{8@8Z{FR)sviAOvrT%T@X-tFuBE zViC1B6~yt~F$Oxjgxtsc0(vYmt8itFNY05qA?3h~5v*K50Ih7M>%d*q_P8v5rh{D^ z16jWet&#eW{(LKoU5FM-#fCP*iMyd2?e~}#lOOYAB;(@w1<*po1#AS~0jf@}u$j*t zzkL!TAzeKoKLPXZ&4f{BsHEvPu_I?khGXk)QomOo`iJ#T)y{Iq=vACQcm33BE5Qi0 zR`g8VNBm$|QSTTYB6xa_<}2r9=8ZiD+&-A$5ea=rODQOQTWW?Kug{$L;Vgc*0mCg1 z`E(`SR2(;W64Fj``_N9TVGqkDZ}Wsw)@jKwh{>m~c@;bRu`lHcY0)VSl}XmAh12DV zUzq%A*Uw@Iz8%;nxFPsX$>}GdsI*x7TG*H=>LKYF54E+Diw0k=84A%rXV|1$>!%IH zc@6zNHFA2piUQ5~r_7~pZ>YR*bbsWkXKZY0IrJ*IYnbs&JvG?SZ{z^D7-0cK{lN!! z%H-0r2X0q5wGWsg3img7V)WcSJgzBc246qCb;`KB(eh{tgL>7yw0gDfJ;>{cv&oYu zrh&TKEu|NiBoIN{wRc^1&yx0IPj`$>m~v;U`Qk>ew>#-SesKbz0`-q{`93Q~%chx` zKDvF$=f1E)c^M=F{epIyeF?H7-^Vo%(;(uR4~i7rFgucl8|vSA+@K=zxu#eWbU1n3 z2vXe5m;#jG=Nrviva4{`a}++bF|Wk6#|ot6Lw;$8q2{FHJ#Ezd!77T~V{X*#*YEYZ zj;QJ4xX+(ysnoT3zr#{1>_a@rZr+#2P!NW#=Gi;rSeV3X9JDYyB}#IW?3K9IaM5k6 zM`4W#|1fE5kX?8TX?uwHjjJ}3n(p$S7i7OMUwO1lIiz`IBqA#xU+Tx<;aL;QM<3zA zS3Q3J3Pv9KkjTaTM{*bbSXoog>Nxp-C%ZhY#4xd2!yn&eRc<+RGg!I_zxY4J)8Kv< zvDXQBH=YIRlN3fCuxGq5TEd=c_f3t#%O4*9}#`;;4{M84m& zV#iK@Cl1_y{n{Wca3o94vqck;cTv=#gD}GMReDvZIgMpq$%y7&)jLm>C7++m%WM{< z^c^Xd_JyAHbl5CC>9G`@L9QxzB#7hM*It^!G1)H)kyrLG6iQ!PnLyI!>}St&Vz$30 zyMH-2(ib7qq>VwcbR(XCi+DNeKy(lrZxf{x1VmM}clfPH6y(d)k*eAY2fY0Bozu{U z9!u?EMIUZ4V!%|mFPCxhc~;pk#ib?^3X}Y4C;Iv<4vwfXJH!Ksjn9jTn%t%o)_uO) z-t*S%VtIMZ)NOBXf>6@oXv<;$2j?g{`=l;KzwwurwR2nX2DT;Sgv{FY<5*Fp%=!N5 zoikyC(fNoj!&l0hz9!UxgAp}~4gxk`2%M}6I-(_&rQ%o$Q2{Dk$Cb~MC`1{I4RM4< z-2=z1M_bYA*O|K`x(qh5w~$$ete;h-UrV={G|)_L z>yC%mE)vmbT20E&6@HcAh5QiS?MgYGO$&-EEuF@g1m9%QT3Zw9Re4_)^lc$-UB!NM zGZQN$D_Lc5%ys_iOv|w;FE0qIY}80`Gc>I(16mlHwGq?(*Ia_xcpDD#T^<0{-uM&3 z&|ptU9t~Q;=gxZw@S|y$4pK^vmC-oVx|s-x(Y#T3dw&Uq+)G7S-}0-Ym1hvkW>Q8G z;OS4C44R#w=R>ptk7_FvfwUp26c~LN&H;sfe z4^ryVai8p1ta+JnBuIK;2&{_S?gR-=*5I@b9;i@giD8W;2__8FQy&Uz7Yzz0r|#FQ z{yVJ51M&1pMqDtw$S!~3m(`eOjkBd-eO69QatG3$ir?yjR)&}x$1C~&n4DT-q}S>z z;71f=Crw2kC6+MW0i@YhRFAnb1_7A9%5mDB6>$An=jOfC|>)t=mI+9s|9It`rgSQ}}g@fc}b84gBqGJCznuZw1eLYnxA+9uq zN!s2eK6CwopaEI;p12}TA+VAY$PtK#>s1+P*Jq_k2HPtv9Zz}`r`$`6JWCOW#1l41 z>Laz`|{?l#p&+SkawKaPu$ME76X5}-C`$>5^7@!hga0J16*-FaLBB^}zt74e`8E9^q zJgX>|yM?y31k=6QL#V!RoPqvpU46z$k@g~T#P>$3-*D1Skb|hGu1wpcY|s_dAbD(N1$kCHaj)4B$~o`35< z+oT&$>v;j{+d9k7ir}SQTH-nMw}$-ny*Zp8ab008_}#!=@~m>6ZT!LELCJkRmcdm| zLRVG)HG^ax)TIBUj`CEBH+ zxW-g=duMSBdBU%K`LAWAA_&k>8?F_}x;s;ZcG?5DwI@@j3@eldj$ zlMm5H5LNt0ZetDox{70mM9Z#+d1U5(@Uwsp8EEb97)z!1_np61^8ss?8KZpP6G3*b zGJd5l99-3GmBjH&{3%XlLlecQ-`69-!TJRe{jgue0w7mqdP~DFss!tLGDe4eV@$}o z`oSwZ()u{1Zi?kI*E_4V0)IhnQK|5pOL&)@26OMt)HsJL~c@z=tMk76o^D)+gNdLb$AN4+^!y+*)p zpO|d?uGqE`Tj;aimoD&M*C$`GcilQ|nIktUe5QgaYHk#~+scyydCYyz1{Y6#zF$@q zxsNJYGOXX`VpW0Bnx>$KyL23<3YK=6?Oy!70L4}`kG%{3<+dZ?E;V|Uvu_09(-5+4 za~#UI#r5Qww5uJvrV8>OMH$t#T;F%OSMxgq+%LcG{=w_`xED&RR6j1q|I__)i|CFA z8@DY4{WU>Ht=h$&MYbTeiQYa%15%_IPM)kIAU;?FCRwf0XXQ~4b83Y za+X%|pVD_{AG%qzU1v;v0qr&ar#k@n3ZW1QIi~d}Jjb)CFmwKkt!WTpf>Hh(>(U^z ziTw&SxBFsw!Vr(uH$2*`G}oV9L|nF&wfgr5Iu5yTK3jWV#S2V&X3R0_Mxr;}jtinf z-CxGyCs}26LW4_aRtJSwRtAln6{vUNM^?yX=OTPQJtms7w>M+ng;(cYUKhs^!Tl&ibAwG`H&Y(#8P8nwwMMNAna=JQscrrER~Am8C0V@4SWa<7=y9z z(E08)9JKHMwHFBx3t0`C4+YH0-;fv=d!!?BTKC65T_^T5obIdH3p&@77@_Ay2Bj_7 zFaSp;tGv3y9IcN1d_JI5+r8t$a19^^^%jYqLVW~1ckJsss-Xp@>5U*Mw>cG+RAi0Y zSf=)&JL0OeJ9fg%ARXKh2z1jok9De1_u zVJ2t1Ql_J&BbI4O#>~&<1+4?c#W#dmyfSo0m%I42%5{Lfz5N@yaVv|&J`48#Q{$4{ zZWY=9uInW~IyYnPhZMA{pj4_sZln83v?y|lGOkFe3_X$}#sB~UQNf#=C-p!iv7{~+ zl5kdD$)b!JEQpkexRieg4;w&))YIm_44=T7T? zmDIj=*6z!gO@&Xmfr$N=ZixI%QCWhmR@7CX{-rK9zO!X;Y64lP;X{5uUd z0%=+5LA#)bGizFK!`bq^E}@IkF#RWo2x|7CKrvtgX+DA|x;)qD9O!P}6ZuNh7gH~Z{iWM~E2Rip{p-tjSVrdih8{mbAhT6m0SxJ;ED7;_VW3dzVS6&U&As!}qqCPMpPmwa>A8i>zgMefk*Efr3vG*nX1An4}^FZUlYbIJQ81!QC+e5k4YYm4ihtNY8 z=~|WGx#EW6xK9XA*X8m=xamyYSK{+}QCj)=tZS;qt?nQ zQ$pIpnfIRmI?{3QgY`4=zR>L(@(xCAMN2M8f@lgJ~7p)}e~^?<0Wv?B5XcHmSN1{rSw_{%nHb zY@qEPzqo8p&1ZXR!I&@&47UnFwM$aUAQXpoP`08~rl?4BWq0!f;1uk(+r&T$Vs3wD z;@;rn-wB|4Y*^j$x43u5?|fX0hvgbP$cM7<*lzUn0xxXxh{q>s+~mWdfP$p5+S^v3 zHI=PH>_Py<13Dl3c4YM%&Twni3$zw4O?L$H=bUKHxeJ_!i5(M+h|| z2WQ6c^yie0x5i9gw}}Y&nG15^tmej{$U+sQd=MYuGYFYo;*-QBE+jwj1GivQ(QX`7Ww9 z*uCOEEN*z>0JIKP0J@g!u>C+Tv5%P8HXk9@;RQZ z&!gI?Qu}hqws}dsER@<~scj^!|A(bR;yhf**ozo|2-nP_w3G?({-~RZk8*qZ^BYXagt)Y>c$W|k%dG<)C&u$yHflrh!9`@Ivk%?$MheWSH7nITsUUK1 zG?egOjxzcFeOkwI@w?COSORrJnO@8gvASx!jw@`vf0-!D7P{b)>R$TIzH#YC)Y3A? z!WE8=7QK=zVM+=~%au_J7!2qj&U{MKAZpHyw)vG`c&M8b?P6>Hh9WCl5%;PLt}j(c z5I^O@4ivsatMj71hTU*84_xR>%Nxy8Bi{+R6R1HcY@v|bmL%RT*xMVv_a86wXM-zq z{r+fxEhb1vH-yI?LTqM!fa-izColMKiO}D+0c+*)Xf1mOhCF;b|0PBTalhecjHmVX z-Uk(J{mi?JX$HJELwfout9ZWMloy)(T5fPmJLba)vpCVYdP;DH3@sKS(f|~$9MsMb z(3dLqnRzeu#`A%KHxw(dRcdB)FOW-xRs_{6d)o%yD@Z3I`n`;Zm|1$b0tq^jVQAQR z3n_49HNpva9yX~6>FZ;B7CPR3_9>j{K>4rhaKKf>$%n3oVC{m6fA`P}UaIP)S@Y)1 z9JRzA=@~kUEA<96r@LqExAIxph*)OE7fK8q(fZa*p*7iQd<+q>X{Q`RbzcsYZ|6JU zlL^Ak#gbCa!U{?&C&(XGSN*eLrxODG)zqa^|3ALoJD%$I{~v#qmLy3?;$&y9jI5KD zWF=V{CxmP=j(sYGBuRG2i0pB)IkNZO^VpkXABQu3Po>xUt@r2q;~zISp4aobuE)6F z?~liIoizZf1N*&0D_j-~_S{nb$Na;p_LS-S8}GA{Ia0+n z@PkG>k(jQ1lI@o?^8prcps5y_$RvHc9X!a2zbC_3{XP5kO2;m8#uLct{kiO%Mz>hm zF&sO{HjEN!t!SRJWO305K7U;AX-sTGTfAL~eg|GDadYr<)qo#c8%Fx5K4;Ako@~U=oD&GV87pT(sUiYW@=)$=z zo8Rq!EsJ@0F6nfB%9cTmNUR}qpZ(vrP~qqfaX8Y6-y4z zto>kMW!L4Pn` zyCG~*wuQDcAloRpTa72tY9*4LKGL!5Z}B7+l;q`PWaJcOATkOPQ_FxK7FR8zUVKlt z?1d#`^I09@jvh($Op8s=ckZt+dK*a^`@QzDYfFID!JLftwWQUkvb3!r95j45K>hVE?9quIu5;ZzwGl@U{gnomh$ghg;c(+ z7~-mpk##nMc7(rKy2olu z@85mcIjocd`efF&4@seCuD3C44h{~!xk4ErEs{f^pMx(G03;fZ3fux-0_X$xs6b?P z!3bavG~G?jq6K0+x_oPb@!rtsso*Q}(#`j#WF3rujpyXvh&7thD7yn$AWy`4p%$y$Fh zkd){C##AlH05Q*z%na2mk_s5kacy&QK6R;jJWEHZ7UrA)^p2Z7Hn8Gvcn}rfY{u9h zsZp<^(6Y!a_q+9^|+GU%+@qQ1sqpQn9sF~mn^(x zvYSi5!7jcF*qI8uqOPE~Voz{07pL%RY|z{0ZDnUp$`qnx-trL>bJi?U@HSa8v-0NV z<74Cr4FG1H5Q%WVzucoJxWK9$bCaAd^X!=ePuM}3Sm13gj*$)&VR0)Ti`bnL;keJX zj`CJ&@!XDIIz?() zol4YoDsuTOjG?zU@Svc%$a;O}%vgdo_~*YXL}|K+E^241;k3L-Hkx8xa(n237o;Y0g5_J*BQ-C zioyGGa!;S=OHepVc}a}fA)pH&d_Q5x2|d5O2C>Gi5IFg6yUHcZRs-jK6AIlh=o~}l zPsF_%S(vm=y{lk{t#;t_k%tFNRF{T|MiT1wRtWa&Q6Asjn?#0~nXm?m?pj)H!!4Wz z4674h+!$QEeF%>6WHheXI@H1~?z^&D+Ie}jqI#UqW?5)keQ^3AW%0dA%4+G|_aRX$ z{MV0Hj9!YK&&B{JY){`=_w4D}Ou?`q6FLSOWv<2g&Y`(^sw@$uJ$t!p=Z{S1c!duiKUu{@LaO*~Ei&P4takK^<%)V2Z-PRYT8)g3H!=fk zx+i07Td})$WHTwBSwMHj5L)oC@>iQ&fwk!vPwSx`z z_M3XazFcmcIfPbjt)ze}Fhmq8(LQtNY>XwZ0V5S`-22rZuk$B^UBk>%w&`AKJ558O z+cdQD`a$?r%DV52^Hz+;IX6+(ZrRI0>65kJBKS<@(F zy64ciFVmxi=DdSXeC6+V=tiuE!$vCnZdi$QJV71knJd?n45bir3R^SG3nk#o)~$EP zTdDD_c8D($9^R`Zn;#Ni=cHw;K$qD%9$>bRI0Y+w(c8BZ>p69b8cEaBYgl{;Yl7}t zi=y0W$SB!=EO;tJ0BrBdo&T97_qsCA!vNBES!bgH|4G_k&p$e~?Eb24WXZN;gYzoNP=xBRNnH!uv$HMTd7#2tcEwK#5hW9!J^1G)Iw zLAeP`EtARoH**7VT_M{RkFdw{@6t1|r(R3b!nQ@u<5ytT70#83c=rb{p5qg6Z~X#( z@Dc#+jAPhsesUd@Yv66aR&Wq4Bg-_Omdn_QUhp2DTQHz&X5VHsnW^;Dt0W}1h=lmm zZygR`n;qx<<>iFOEBnXscQ}Nn*0`$RkvRdXHwSlrI7`nzbr!o|h4-CJcZ2OkhC>MG zSo04gyAmW8YI;f&UjeV4xIzUb-Cn7t z@fX`@*bj`dOpHTcS(VL+G}Yp%qR*n+zW3d{&y+dY8l*RGkle)909YQ5?4^k1P5b<_<>!FK#VPM`jD}?A z$UIl;d%DQyz{>aSGe5s%CU7{jNAwHV{7z2EZG-w8u~3=k>zT&st?Osv!!$2kHNNWN z=YaDk#CU3rbCcJnZ@vf))P$($Fd1DH1`D*Mlo=U&Gs^i!-!baZ98|r-n4D=ot9`z= zTEvU>>F#DEqkH>0va%b?=zcge->PZ)jj#q@kvj|9ta|JFwlWZ%=02!>Ic+9+4RYq( zxlsZRBrlbzu3XiQiUeX*3d_p2xSk%57QK5iChkKu8!0*yid*-@y#Cph_XbSJ6xqZF zJm?o8Fvk?RH%5AI;zJmbLMcmWC%uLPS*guR)9V*F#8Ti9AGQ0$|J9SopW_)(bL~`P z&<#kxIfFVpKygjp_`|#F+LJ8fZ47cXj2SdW8vO1KVqq9P?Zbr(*HpVZ=a4(prr8OI zXU&Ba;iS8RNPYZ(Kbo*@CpAhy6FTpqw#Rr0{%2`2k#&c?2OHjNp*|GISd6!AepLk_ zxt`j&PapzWyb9>@yfsY8nN|2SJYvC)2WC@WT}43RclOtF8BM08FkrmFnC~nwIem$j zd4*H>cNNo{z2*n4hhQIE5bU>xYOS=*cw1d%F*9r`&0Gh}{BA98c0%6NGEKB?DRHAwJj#&D zgBh@gERqbrf9U0bunKF{r#$RGX_2ji@^4i@S{>gxjVz+l(n<7v$vDZziq3 ze>#w=-)IzVlCc2^>3mL8uN5Z8c2(F@tCEkrpGQzIv!-yi{9Tpnx2X9;=B~W@jj&wJs zVn~KHWTnR`JqR|0^FU1SQ2ES^pNHz(#Vdl*l7b9IW&sHv#OJ0lxbs@DZ_+b723~sv3d;*61#Cc2>%k7-E-VJYLxB2;6ars(1Uu3 z_M*^R@k%d;^MWKm*JepZIsH_X6vVlji!U}{iFRDXWlgrGEhCc6*xOPr;K7}G;&MLz z%Y{C`(^t^Ho*sfIWvMTFgL6JS^_zPg#VD-$w3Qh}T~0=-O&^6xSxM?Ug@RegbEV8nl5CU=7begO!s$XA_QzH`)4p0FLiD2WJ2 z2o6cBMA7lwQm0#}Gs^Z);oMnYzenWWb>2lasQ^&=e(|Ih7} zyq<$pRB_BYaAkM?Yy$Yos)S4L%*-k@sZ~*%4GPkHn6=? z>{DfzGu|1}shfez%QeuO$XW7UOIA1!n4I=@|20B8FyHHYf&KezLHRN0@{^P$4dF8Vmr2ZYq0(^48TB#Esj71j z^8uZmuN&2M(W~~CcF=uscip!Att7mK+1e+YFKxBXg=%mNRaH&R8Zn)H@EwWzvuvyX;;GXjjnI+kgaBy;v^k`uG(Q z;A*aO$&aloOnhwfL45HFS?(hRlHddt{xDIl$6N-2tZ@%C5HD)v7;9uQA{!f|5HieM zTsJaLIr{JR$})5KFQ!NZ0a1xyDn(9l_omogr>Zt!4~I4tpMUiS4R$A6-RV*a)@I@2 z;p4x{ck30_{$!k)hX-8$s`J#o(D>E*`In;zTY5)*-pQ(SFl4ECRc>iR4Rc500-YB( z)PsKF`U@CWNA=f$l00HT##HW0!LbisODkK3P(oQ5g5~9ys2)2;$x#QlCt1r~p>hFb zFIXD9m(RG($-z~vS!+b1?Nmnsd@CT-?(u@ z#8%(Z($c}f;pxbyyp0R73%`THhRrmx3Raag{Qv;0A>fIi4qzpp!4MGOgWO4~l{MWXPw=w=GGKz8Qa&zE|BtbtHE2x~VT z*Qj=2=YFjws?N^FM%?S5FTT>lwtt-Q(_P2;`T4nQy>eEEv65G>{yF@!3P*E)p6>hpn$i}R7#;*;-7ar;An#lUpPCZan?dWL6yEZlk z5)+>~_4?lptP&zjRC;_nPo|}%C45KNGAsKjPTfIW{pr)EPo8{+90lt?DHL|;0MQYS z`6L3Zn`(C}z66Xygu_wu8i?6HLzco9%00^BWIN_KBdDpVVNvn)6m;1ZG5^laZUl7w zW{Epyxx+x*ig&;T>Z~BIKuvYc8j0|lt|OL~uKxF(j@APowJX^f&OR&kW!!bg)jpqT zpK1OLAp+OP&E5EcPFzdZEfayL%C#FSap2U>9@F{ZFtU>#8mgYVet39D^y{4)vyi?0 zURfEyDMlzVCl~Zm)T}+G&}($BriQedA+c?Ojk(z9gn9nReJfD;lzL?S-_HoRQA(Q2 z$XL^cFx!83nP78JS=nhfQK9QnvWmrQ2Zzw}t6MIADqI;bGc{HH^6}=iYcGyc&94bq z?fH#}i{mk5RbczPAK8WB(Sibkz}>Ig-;|JK8(w{H+P983{^%dT^IlZl3S1j+9u4@6 z_TOP>J^M~y{6d3YrTilLkLzb8tD+-N6i3HazdfdH-{bGw!deb)ZxH)L&|0@K6@FbpS!{~EXrQ4@{4Gse{hM5c^nyihY&LH37;d|!D1iUu^ z;k663gF>C^>)D<^_lydQEq+mrZTfz_HNDRI&m5r`*~^WHN)~bGnys~~=&XH5wBTW{ z8o0Ts>3wie|9fGZxPRl`(GR?PUaR3t+w(bA!m}~fcu*p?b$qf)k8lu7+h}D86t!t- zZ#XB>2u|o#d3%{@&1Zeq&9eA6po!cOULVl4VTq)p1 z_U0X?&LjRxaM$|_n-3@_N5?@oZ(6(WEY_umDSWtDssZ!pnu-!|dkG{77uCIKzo3BO zW9P@~Y{!*N3{Mzp8sOH7%TW{Mv{$Y~N$_KUPE(z77aJ$1AGkjW?pI(Wm}9sY?@NQ~ z{o1_1l4fO)s7{HMr-n3%Q-?vkjGZ3w$P!)P3SsZr7xHT8?WU%t^sqGRi6Rz3CP(c4 zrtmTFX#9Y>ezVLAtt#*4rt9Xm62_LF?||aVyJOW+ zVBmVs{Jm+s*CCI7#ePMaxZ89}1$Jz8^-Hv%`GNe2uu3?cAg^8S^whkfspt+$^QUIe zdp1jgI{cbv)wZ(%y@Y3;)nHB!(YMM<$&|Co3Boqp^D1s`7-HhhKPID)iR>ABiLKl8 zI@}wt^x(aH+kSnlRKC8R6E0u^R9gRXeL}0}3T2yDV^oyPdt!5E|Fe->#vYvo??eAvcbPOYh;ud(MvaU2Vk?n=73DS$Af7P$>=2v zHF|q`u2R#+mRioet-$v6^>uxGRMqq!EC6sD$f};d0C#c`<(94P66V$nvx#i9M6vP0WufY7y<%GzML(W^q`k?Pgyyb?vvLm zwl*kTP7VmLH*5blF#STUgk`A}#*DB0J&H_gTT31pXniHKT+0oG*C;1Zkg!BndQL|A zU3G1!Nz+u6n9bAs?(P7L=}YfrWMZ<8{~bRL(_sti;|9pptc)DbfeC3w34%@*d%x#07>Nb2?%*a z!*%7d5FrCz_Y2y#G@B%SE!j^2^|Bh^aFcmwJSQh_x#woVM`l*mES-Etad8pn&FPH* z`|@KPik~c&v=M)ql9ZA%+Y#ph>@*2g8?wMn$<|Qy-o39Uz~j+a<3JWxF1i_SYoi`D zw9@5l^(}N{Tm2e-aRb0?AR(bOWN~h4>Vc80iL1`g&QkASwq9&(q=k@YtN5oP4CCv-pm(H7H2LEL(qx?05V?tYJGoeBcD@ zk5^iuOC1;POtbf<<`|Uce0^3ss~D!!yP9w!P1v;z$`AQ48t-KZ*9iIdNvPMiUp12t z>Do{z=jCcwX+MAYrf41v=BPwM-q_G!Jziw9of=HRruySJq1u6tZlqo8ZmFG{E4J$g zU=Fv6UQ9wlLTEgF!pFa(+Pr6>cIt)v!3XKX@05y#rw#fZH^WJ1&SLqe$S#rUm)K90 zqY&jf9{FkIu{wJB#+Z4PSV3(wzcXhoPM!J_Ns-xLKE6i~NR>G%9;KFGVPzE@7}(MB zw9vUV;`1>3l+o0mm~;&GMAm3!qYc|Hke710Ki>?mVT#j!8KS(=_zKz8rK71?2|Q5g^;~nw1R?sj$!3|D;V?g*=VWJO=pU$d(NzC zR&nIsRQo0^pc^;J9A@qV_lN*rCMG729(8%3R63hG+M8OU?wX_?=T@x9nzr^Jb_xwK zK)B00MN62rM5x3D1qB(;%nUzmZ#bC^g}quIrQX#Dkl-OtiNBaG+h@Qwx$|bc`z>if z4X72-yR`ZORpPp7>?S|#H!_lwvtGAfhas7#v>Pe0Ug}v~y#8l@>}m#`&F$<6>jvd6 z-sPEVn_*wSeg)jb8$TDh#3=DZN;XvFcoCg`b=%mvkkS{^0 z=Bq>bUe#~T{c>NSmznOF>iQ3pAyFh!o>!MvP*v`8Ew6FsTS7jEsTxfg zvBS-5kVQgFOhBL<_ycNIHoj#huN(>@QDDGE>w|n`Wsmfa+gwc@9nlhQ1>pbY#UQ-+ zZvUfHZ6&7SDNQ-KsNum&knX8k>O?#q|LM~wyDHo|9_JzV<>SXZfPjX8aZU|CmU0e? z0A28iTEp^D_M-HXmJ%*m%NtWY=gH{b`unxVNFHh@z!1QC|8LY5N9)Rg@_&-Z=#trL z_xf^C-+rk=Lw{AulQA(9tk2EPo(?>#ICpTVkGc(#=hm94nJM)yD5N)P?KoNyxM247 z8_WHf{$zAg4qJ6_b2Ya22_H)ZGV{LLch|`OwU#a$?Ue%U8a7W!XVe3RR-SxeTBaA| zasU9sI(kQfH}^L1{XMh1dvKJN8k7(kT57Wrv4m<5c>g}{IIa;s0jVBhs@ae5Vkx$r zSYBI+yUS!c(=Z**sjV7wM-k`nXX%f{20Ln`#LqE-2g}@go*~3J=U4v8M75Gt`=fe6 zH?NmCmm`v}O-sFOK++&3y~LxF_iXK8*J>~;TGUA9k$v7VgTf{N7orQ(OL>+&C*ava z5`5pjP5JiCX1vVV;^|Y@=e`XwC!r})3b9}c@`I7DwO7pCgX9B*r zC~blFOYV2kmvb%Q9XTJG0-7jWBNyfuxD7x57s7xsSM5#L+d)xMUI$OqI=6?0qADt| zH8nM#quHgarT<#kqAhOO=!so@8&cYbKp$S(g;|d+F;R7ZYxpV&fIgs$3^@+;z1K0LmHnlfze9SwP*>7_V{m2*P;u`yn z{s5heu@9ex!}@Ilk^sVGEr#l=3cyQe8iHxxT{KBH`77z~q5ltVSPRlJba!`Gy$Qzb zcI2)sFW04$INGXqd{`5YNa5b}|0NcHi?HX>pwHovnopjLRl&ewV(v>l-~7({Ezv9` zCMN!6E#8xQ=oW1HO(sM3aRqqB>IG_bKu@2;0WW!T2)5DG&7`KMM_1<9GMgSNOt~;W z$HvNf?(De{@GrO>;$Bsi3tU~;9*115c;tbA!#Ua6E!JX$+}zyQ;g4nB$o@4g{4}kH z=>h#2%I!e31jCI)8uIB-j-UPn5?n`4j-z>J#OKd_{o{+Ps~1T~TbtU3K2|XRC&K~V zgrw^@?<(XY2S(fvc9tqV&=cRw6GMf~JHAv@R0vrOFtY>?o_5u77@hlXc8>u4zqldL zBT48O7Z+-@YPlRKBqVhA?b}r4xC$48J%voI_J@kZ7m1I>8ju%$9E749s;WvYOHIs| z?)?#>mCM7!@w9+hh=XI~FDV|`B+{agV`?R4$(6p-@#oP{=V5=QS+r>+PqYqczA&eAf73O;K^i=Z(*wpnEL# z;TPeHmXWF}@2Q6)<|{9jScY!Ot${5g-jERr_km6;Yfq>3vEhPR)7@e_^zI&u+Cx>f zPMw5GDH}z=4u^kxeaPvm%<`3?ZW9lsy#Ztf=;fvMXi4+Y4>o{`ZDcehfqgQu*ZSXV z4XEm6YxAKEWaOfvvldpJ=;qd$y7IcD3Tpfb89;1CwU@;<0y0L3rj*+jGoys{^lY=9 zu+2lYPJs5m`yD0`go8K(gQ~-EJ0-H4)*c6+1KQFQ z&~&B8#i$_!v$LX{obSoh1a#fx5Va}r%yh?mWAE%)XZ2zNDtitD3%yaqr%ye#{;~9u zFE3qYQH%c^7gm|?I#WkXLFD6e?9w)iP-iHl^yOSvwT}>++NGUcwP*p;{)~`!??}ws zl^M+b0;4as{+AX4{Q+ozkMMq}vftlG6g2N}c5$(ciHL~M(%0Xq-Xgm?L(%3Hz83o{ z*Q^1zi>r-}aBy;RwzRI;PTaV8Q>VnfxX2wZ)VXM+zs(Lmx2QC3z^uw^#hj=&=VDOkNrrfa$NidCb<*f|77j;-Y~ zqIRkRin6t^SjtZhS9_|*#8eB`DZ*+e;4JsDbO9Cf50*TpGU`kJ7dZq<{NpGud<6o5 z$jVw>9fkwYTQ$wG`4fxPK#p#7q2<=>bM8OtH&7)i(*Qd!_aSgB)I5DIP^enG0E>tS z8`!RPXGtSa=&z;yzUxfB_cb6&GBen|ok2+Li38kMx0~sX=RT^*&ce+TUxQ&%;IQl* zynLCInVA_;C51{T=a-1I~s*e(qk&HHN(#>_U1DO9S^Sf}V1?6&Y2$A|u3Ma5yi3mwp3)VD4Q9 zbjR9cSJzT+YPNnM2b%}(hXv~F*|Wz;qe_KP?c)z5n}fYkJ385pyp68+JMTNC2%gn7 zey7?pKK-j(|I+zjPKONuC>E5FTYpEd>-lZt!9w^L6UEUyQLD=t<=$Q>#3xi0xEDD} zO->$P#w;Yl3){`m1R(jJd5!=j8>`L)!ptB92Alt}2nbT;4!sT2!Lg~z(bFd-J}{Lh zl|h9~<}_~XQ=`{{W*hni7ckZUR4h&{L5dq-m)zWyUWYFC?mg4jx86HgZPzPq`@&!Q z+Ykw)m@-*Al&6-aL8Zq+jt6*x?9zoO3FjX31g}!t@e$yZ)?fQLw8be;ww=&%E9 z+5JNO$TiIR6wcpAmI!EvtgrX;la~kbGbu6go`c!B*k2hItBv+sbnDz!isXq{9fjr> z?P0bTnwy&ekO-uV-ZZt7h1#hQX{5ksi;ioIAxdY`(`1&fI1dtwRz8FNF!IUTm>TV( z2FmtR{ynwd*wo_PxAyZJB+Mtquoa}4Qcs0=`1wUehkpAcffARRnhLP($hOIf6NcsQ+04ZSC(`}*>_ z{+2KEmptM*HJZFc%QqJ{Cyf<06xz;$5OKJ&va*<%n2eTXe^PYd5A;MV!UYi8!gb*q zX6MeF8~*KDRTaq5QIEZ`Qak*9r*rZ0p}PkgD;qU6C6Fm3+3()HE&87l$1m~=PW2M{ zywu;*WaCOnPQK?BcP#;+h{f5+;ckbymj11GTO%aYK$UNLW_q4i2S6HC#+Y{{`#Qg^ z4mi-H7o*4V;mPOEt5z>uo9fcjgGB6xKNSjaatM6V{e#Jm6Ml$8g8U9SNyV+tyOS9VgQCqM_s>RX>Bdxd~gViC|+#0qH5mbyP2t#-J+WS7j@46opkU)xY1(N z>R2fp*t=w0G*3a|*U2zC@%gqmiWkGUv5pgR0vU3Qvsa=&wV#jPFQ8Jj{%+Dbf9+ME zk*I3J+2G(X0a@Yz2RP7Tce26w_>_@R7@NvSkym3&xUQn2qVR7$U&=G-HE@!Qng_oQ zCE&KcWtNG~7|%dGw>HhI4!9&F`p8a{5J8Am5i-8wOP}0ZsgNdbR;tB*E+1~mZcH79 zqMXDMz%WBi%|dABgw-JIRu2H)-~xih*24w2g*2IpeiuNYvW%ValG(R&hu}~Y08iie z_yC;;C2p}Yav6f_?h1BEsxL4Hj+(}daIfpJ|61H2DD4%Vu373bbb#~>Rwo7cZ)*+ zFcZpA;Jy_RJMZUmzQZ50j|3=~^#R~1k-U%z+}b+ayaO;?#R2CEzn1qM7bGQ58hy=}2vmjMGBD~NV-#~`y>RubCsYt@Q;MOKTPJXknI7@p`O1U#2B;;gwMlHbQ(O$1g)sY++uE}AJkVo`Di{2!9 zX9p13)|*0^3aNREz#ddoR1pzHKL8O#$ho`3LGf2^!17O{chSgFFArc7kucbSg`S?W zwWpa@LlO7rW3z9*Z~cxP^-m6*s`?2CKz@Szk%>UULb%@Rg|v8ke0H>5)^I|UWx!so%KL6 zEb}@Xm?|g`L?&2RSP+wv8UV^|nX?tVKnr-%?_>_tg9b*P7QkQtTQjMx1<2uCtM=;t z{tU3d-Q9l+m>aN>;`MNh#@+iBkIO>LG>6G~k}iV~ki#+!SkH+2UR(6mRsr0+^X!?k z%yn!$dPP})o?Rm!#K^?>o4Ntr+|H$LaNEHg1I_U8%F4=a{`^esT>Nw^FDnb%`Lk#L zXL(cXfcmJ%l_za%6WhKoe9hzn6f*t+R%z=H2mz8FWmf2%<>lpuw+$aZegw4GY)G-U ztaSDM=FD)x(1_p>j8_PTxMgi604@TIv|OefCk80c+_V>knd?|S<`M~5N1c!{HXuFb zke+UTp$JE7kB(T;?eq|u&Gf=;4(Vc#FW$9Gv{-4@{M~_ zRibruPu&=mJQ@-=Jm`#bQ|nwuA;vN79YtwsK0a^eJ7Upu)R3c!hlE7iNuWwbFh688 zW|zwPiqdXy3^@<_xgiR47OD~)T4z3avG_kE6B3f_@%z-kLPbe!H;Rzt=YM@1o)6=| zLty(%yjC;f6Hxk79j5_NCxnNWIv;*GyS;!pSwO5MoRtd7aasD&;TmdVF_HNylZr;I z51(wt4y;o#a&!>3G+hsbE&Vd5Qh289m(THd;1H67nUKWM%@4jrImDP_FrOwb2v^a^?$dU+ywVNc@&g>>OcDORhVM>T}&D z{;dx|64sYUNaF;d9cc;PMOYtGpwsy8F>|MXRT2B=e6px--bj>3Kc5*6Xr8PEG2wsy z{MpZ`eVL9&95&_YsG*^wGlazeH<7*6N=ai222o$T{`^EaEhxH~;O35YcJ&cf$=82}U3*`~YP7#!;?q}eYd)Q?7ifz^r@uz3o|L_l^FG;8}^ z78-4|d*Q0%QesVb?Guvl1JM3>49c8ZQ@JfrY4Y-NSFHYLym~0zGpUy}Ap=}0ZV@gb zOSG@aU|a`fInW=)5QO{OqoqUuYUJeP?CIh(Msi>x93ZOxl#c0s^BaEvE7_f!>6f_&Zb;frx|=2k9<-|k*4v&om(b0Xg_)DVw|0yF%Ve{u#7(XN?wrcaBIN9%N>E6Aw2s8yoOIq#i(T&0j z3X+nOe}^}_VQPbznAoFY9JCW^W7GNL=c{iPN`zV9RI7j>_U{=IBVV4~HVSZlz znbikh&8skc4NyqC2q2`1dG2?KAU%7#hHjVD)z@c3s=QkVvf*YOw=O>`@;GV>sB(jx zJ9mD?t{;UpTS)IsEAwz?tgYQ9xmaX3E?{Ez?qo4R3M&FkV^x}*-$RRbDn`~a{`<5U zF+54(1@I`31MOVH@nRUvbgp}$vNEN246`k4Gx9k+G*r~~OX~0L4y?#*km}l6XGa{k z3b<0mYB2Y#ZZf$L8 z;p6}|`w9&u4qt#K5wQjuI4%*BZl2VJ{0aBHOm6pP(l9VWHstM0hn^>eZUvkf22CcM~PZq{Hud+y9{MhX;!Mv;23One9oTefFx z1JyR~+26F%&g~e@Gtzqa5IC>r)8->A^k5ICkyDmrW0>8wXV$ z3=Kw0{O&&m8Vf%mySsHEugwcvPp)$pi0V1| zh%(1P5um5jeF7)JBgF#-!$$YkC_a4xr1JlnUV$A=%N_HJp^YqW`ZO4v1Ikv;d@;-u zAIX!}Ix_ndpsrOeR{lYgo>S6k5yE0NzS|Q+Ztd2cCK~S^OyaL=6!z8(tWYpR8Sn@*L6Y z%Pkp2<~FZSy=G@;m-q9FEwUyiJywSaDaeU#e7{{42k=Hq@HFNEK)Hnw4h8`-dGW%9 zzi78VLDjR<(ez>|nYjqPB)$IX1$;p1(TM^_Tzq`|Ok)UYqH<-Va3zT(e|C0ueJJ0g zJ4yc3shBx38sT5+a00%o>(TfwZ55#w#F5ECbcdUPpFv!F0S@sJ_Lli{vOz)$3th?t z+OZ?<+ty|V1n|LgX%I=kbP47MPJ~8~(%xEbM!NUd*s8r~s#>g;j?U*QBcK&TA29Q^ zqxC@w$fjgw9=^7tqf!bhp27*;v&aVDX@-&od3Jy6j@UMwvB ztt}(*Kkmynrlj&5xHhsW^*pYvsrlP%)+S>;Ok>hUe%xqCzLGzfE~?NcX{Iw>1I&~X z2S9%m}$fNk6nC6whH4q1iC{bc?_gH7rQno*bWX4 zQ+C}*Y7`y zii*B{W8vVioT~8y#03x+;aSZ-aZKtj!QG^bIuW>cHzT9vNSEK;1$qa8_Ftf7=?j3n z9{0HZ@}T3&N_{pN6f!hcAG59b^tWn)&}P!5M8YzGUh?=vXLZ{h+V0 zE4{El{1YnWC^Vmi_#Pg3*bNSXB;B*hGRjIz*(Dey?z;5=N?N8?E|74Wj}kwK3868l z7AH|^heQMwQUgoFrWXArG7@OC6|p&R3jV9*j12h}XpV|-XS!{98Z(5^ApI(}<~G)| z`%~geoUYs*Mk6DB4q}oZR>nHHwydx`zR)WaLBs!$rw__BNOa4aP#H7Q0V~D>fi6l) zcB&;HybHFRwkyxe%KA)AO~I*nn#T{^^ko>?rG?fFy_+lbgRW%SQ4TWZa#%3SaG80V zsXS=2BZXf9;45Tx#QwEkfl4^W{2^<#b`Ej_!_A10)b_;+MN5A<$sERxo65~N5G4m3 z2EJ2E{a8snQ-2-&Ij;o&rMlr|?v_Ak6q1H3>l~zC|1IF0B-p;L(cw^Yl!%L$M{04o zc0}6+?gu0ZcNs)){nUBaKW&|pC?sT`5#b_0FLX8Tj$mZtyQf|tzGz81Mpp7c=(%6> zC=SX!Ti?#}@WG+Rprv_Atntvy@P)JQJtGK+mg9tyJVbU)$$-JQ@Tr!@Fwvu0cDbD4 z272yFA4wa@cA^d`p*yl~?aolJs_0v}B|ESRxVSP1D7Zf`@7C8NspCr8lQnU+VLUp4 zq=(yEkf!rW?D%;h`jgxg#o`jKBZ{VP*!;7mQaX)k39qGt@LSV1F?zi%=C+u|1o0;Z zNBN^`a3t*X2&<~26ihX)vci4h-s8TnJ3&4#o+A%%&pf#+w$Nn1<^qe9vKo1WW(uK? zDp?;p;9Z1;S|uHJP90|polT-_-r?;IcM8^UAKxa|mUmA$-C60W1z8wgCUt+kk{Ntz zyCcg8dwsdX)z(%a3M`ljQf|VMd>3Z~Rsw>N_3|_rlKw2gVzGSQ@m!hso8;;DD5Ajz zQQgx~QlWfdAG@;-Iej&+v64H21>vTJy>E`J64sL-+;`vW+E<6oWyPDqLK0l}vgIIa zht*@J?Y+PsIjm-Z%>*p3XiI$k%gby~9|c45%zTWc6i7HcV9^i9J9CV)!Bk!$l>!Q7 z(Wd&f-{VEe`2)N5qn6-}yp+ziG>(3@my=@rM>E8%FZbM7aX0D-a55^|Bhw^el!;k} zUZto$nnAWrwhT1z#X#StRgW1BZQnf9$0FcLscPHOgrSJi1se6hwx$|0D-K@=dv4&> zAw3~B6}2L|ChzF0nCB~H$QfOn{dDEb2Oh}17pge8|A;9?d7b$p*?45^|Pt1#_XV@Y~9AS$bYZ0*wyuPtmKdmB;vzfe`+=mZfrxFEKfO z+B!^(TC~d=7>x$EII~8cB#XAS-M`8@aFo~ty%^LzuouW<>CCB2!aJU?UY1+8 zJ7fG<(#&`U1Q&ZvOOfA@MCJApvmEr#Z{P1?6E5m`Z^110+`2CscZUHYGRV>3ic7*r z-aY^8FKR)0h*Z}6&S@@7p;bu@{q}&+y2Y_TK0&yp3~3)W_x%EV-j{?Vra@(x@7AtSEG!vOx9jQ&{fuG#nT> zpAt%DTuBK!s7-0Gwidj;-FHdoPHn^eH~$||R{<5(_VpB@*a_A5cq`O1f zM+yuxl#)tG#~>{o(!$U+gd*MD9l{L5%s0OO``%mYEEa3l%)RHHyU#v5e!soPhK+*N zH8e5u*^%aXsF;_kd=&}4#@1_vr?6C{+KW*G#l*gU#9?)Ti_TNBJbRYNTl0t6w*$`? zp(tm0We0z>ReRt`F8S5&vF~)$qc?aquW#Qyixzen*6dd3l}tXk4V^+@0>po`?l{fN z@2u9AN49}msI=OxH#z?}CGoTvCUA{0EP;pQqK{$3yU)W1H4>^t{Y;EhpGL=>J*sv* zw!F0&gf9kuY2#Q=%PIzb#m0=eq(#T;>}YicdJPXNy_5)xj>nP1b#3Zblg6OvL3&w( z4)U)o^ncXo_@zmlKm~d2lb`GbG8hmGnUcL=t7I;>_s zvhy*Jg;K18GCfXrq~C+E*KI^R)#@4X6WQ9Ie<;D=D#8A9jee#lv*tgOY6C?ByS86W zBCE(*9d*IE);8V^Fqb%kbYmms!9mi$XLht#UT5!yRoB<_t@tI~Xu!zL9xBbr4RcS7 zV1hN_Pg*?79{Gv7qMQ?>2V2Xy#Av5X^z4Kco<#Fz{A9CljBqlN{b@q;Q~BdrDYd9! z4ggapXYhuuR6uh-i3j8Sp2`3l6gj_c>1o+f+8HbRP#SiSC-!&qDPU;MPeLgSG9x|~ z6fJhVmJ?;c+!CE3n_M+Ltx%EdNaBA#%hk=V@}=@~o0PCwKCW)b+S-+PM{|z2IM)65 zRmuNTDP{;2XA2A4DPt%(ZOvA(V|k^#*Am(Xt+CX*@ejKjW4SnLo>o+En?4aYeaK2l zIexCGH`}1eMNLU<6z=jy)i^!<`E*9y-NZ@g_MC6ulVN7S z(^*6G(^|UaWnW_dgHCtgIoczbF`=--%zhAx9FJm?^-Q|c##UtEnPe`!O zU11uR@aDqy<>_c<*?BtD?mzZ@JlSc?JNKEBBYg+yZbTXA#prvI)!B2wDTM&m9&zB` z1NpnJDcgldDyV68hNTAQbzYDvp>5V?&I26g>o$Dhuf7oK&c@^3)K*P7k~x9y#2A6X z?E=%U<>J;iA3SfcWeGLeIo~k7#&+1#o)4SCl(?c$XV4F1HhkQ|_VBA<*Q+6 z<39>^q+)R{aE8cRr>11}B1j+NIW|6gIWK1>D-!OWz_#yI ziuYes*o$Ho32C<;dq){=eB9&gs%LQZs>MVAi*=}o-w8x6k!R7ipAuzU=W3f}&zW$A zvBUmeJ0>8oG_WLQ1(UxzydNd*a%Fo>pbL1x98pfu@W%0<<^8Z?d1GjOf~8fhi{0{s zw|eZp8a8#Un?r)YDBjD`*7Y*X#lECB_7X5O@&ET(8k;)T#} zFZEPYWHHiV`SIQkO6gr5TfkS-fq40}`p-PTnm~9T`y)M*bljGbZ;dMbT*ZNhc=Ux=3 zlbQQ^EEMF+g0@RMO)P=Mkwp!?4o8gdAF z+DT2t(H>1`ogkeM@t7p`aW6A#bglY=*u6luQqg(FC6 zy>662RBL^cKe$QQWNv-XLy=OBb1>kg&@8^bx30ES zB8T8PZe5wORIsyiE%o4^YnYwN@8=t`mR?znh_U;nK8cg&i!v}#8Kvdh-k%jLCkpxe z{n?&cER?IgA3V(|Hvve(6R1{!1im4@ek@())Qo_53$t93roma zh8ad$O$(lid1Q$rH?~epo3{$sFE5@i zm-NUY8fnZVOrLV5kkNPy3R;DpouxGaE_aH`#T#G4wIk4j5X+9cu9Ew)uy9or^!v^7 z5t&T8ou`M;?!akymS@vQ9v0DRwm4TRSe|HV-QnMde05_vLpmJXGjug6hCc}J^HCb$ zJ9sU}H#5^^i?dK>-(xLDDc2B3(u);N=*5D+z#MB`o}i$_!a*iCB1Z9;Iz2oN++6$h z1+=CnudpyL3L7%+uhR6&qScqP+7c*LFFChJnQUGHzle{$hH zZwlzJ>nxL@q7ZRoP1+|J(jJ|mA_&h5_a6Xp;2SCIJ$Wjn7rWN9nMS=mm6}qQ$v0J> z9zmBj&mt)sDj!(}yYO@1EcDYM(i0l;ya5PU-Rd+F2DUW3Gr7xETjZjMvcX6O?Rm~G zeF3YqHDxT3Xdm`xw5Yu?fU$9%jH_pW!6;7}G`~*L2^_kSr_i3;Je%Hovx`Z{F{uOE z>c}=}9*RE~MU^7P7ju67c-yQxBZBF402mw}8(m0E(F-cIj37dt+1-@Hja#!i8uQ6= z{UExG&D&_&7=&goVf}aD^j|m5s`lWJdo;u*FD70nZuzn2(m!RHOxxk99$%#fCbg2f z4P1xM;>?9oetKY>=}_Jg7CFB7-M;Hwdj(loR%eQxAtxqx4G3IWO;ijsN1d!!TXgts zWIhO2C8d4gwt9U5dpW!eTpQJL6>E5sL2Xh!_D=LUpVCdak-eLFyUl577jEPONm)4A zAuLNK*l*w!L$SHY)#n2)Lip2V$FHCEhhrus9rhk~hTMxCGVC*;7M>-I*z7iHueaNB zaS2^x8nS}Jxo9t-FEEX6ST(;N?er9z-@`BG0M@+N-4sVMrI6Xu*+O+^Kc8F%;ZxMx zcfNIdQbijYl`%y5Gm6OG)rrJb%}X<64e}c(BcR=mbQM3ofkNBIJ_RIDy6?6)E&2YM zLhMfG-^MwoZBsjtcOIP?DoW`+TSmknZ#{&3C+*l+M z?NEcUs}JroE#ti`0;}|4IOkN z(dL~7ZfBL?_gy``=!hJ5gCb|r>Kh%3{4{BWJ@m~L=!K0w|4Fksx)0S!bdE3Ov;)`9 zKY!xcdBF4zc%GJK$cGMu&*6^cdW}x%RY90H%P}QWTS@L1V>_Y$WYO6Z@O`Dgp6IQi;t5mHISN`|R{deA>_3J=r1a4Bv{ zs!N&72@8owaWk){;*XtYMMv{`4Sb+8P3qijfzkkkM&zqyz6PcGHB@pU_5E32QoMjIREKcIqO$98s z!J(JLh2+wb zpm%{P5EeFp=3_s(VGA9gL=D33bQq&tNSVeI+xgsQM{10(*j*)w*bSbLV@{P#CS&|$ zzuq#Hvbz^j7dU3E%?>vuf)Im-_fy}!_BXjW!%{M{Z(AT| zlP&q)4Nb*A6Gr@XMaHV8b8B_VfF}L+@oxb?xqmnrd->KCZfjIvHw4Yjxs?5HVIxnT zI=H;R#WT{bOZFNsBlfC|A|f&Hvl$enA*`5)xZmSTIe6X_wcymkX~t@(ju(w(TNnLa z-mSJU^5H{d*>Kt3$*ZOiS5N5i_ch$JB_u{=RGHMZqF22mOUV2$yz8#Vvm~1IW?m&% zWp17VfJX%wGR473^OweAzp-!Cb^bf%fjvcvDi10vE8wdQWUN36@fGi1saIyypoY>_ zxivOzS^6}3!Dk+pZ4OgO27StX4F;kNQvQdFmEMR#;(WV&dbn$u3*g;VFl78X8M5@v z>#WQ`!|2f#&4TjcvQjPWk8PMlW{-koWuxIx#?qgrd-i{CpF!~pQ z&b=-k?=IJ80px-zZ)qyO`hD>`7;xb!q}->n^mgZJ>}+&=)ack)4MM#!8l#z7U4e`KoWW+*84`h=Y_ zZPZ1W``c7`pDV@;7re*oYq%!`eyqT3zmth3Oz_o2-i;mor=B31!Yu2-rTSOP2KgYZKGC#q4hD8}iU1#eriv zAz?6mR)pXqz2siZFI}|!C{1K3sZ1{@n&n8I#PIilU$YC74gUP#eO)sn4F}F2T#?3U z+7FUh;Rh3nI_C5#R`szvgsu==zOAiQ9!BDhQfU-gGoedPg zDk||sdwgt0vz>WjhYA%pr_p#k8#JU`K=Auu?j>q{;yi67gO>UU0t!GA@sW3V)B&>E ze0mx!0`75)ES9?t$U6D8yLCUV9`M?1!Wc4f!K$bq2?;zKDsJ-~u3$?Fb40-LM{VAx zuE%Xh`@{3|WR4@_iz(i>zR3e08ClOy#(9yOFlB=C*w`2tvLou-W&{iFSf(^6Liz21 zs&B<1F`lGP=~|1ptosY#yKcCm3hA*RVq7Kx(^Dz-oRs=eno$wQ!1ez0UiZ)NK;M(i zX|%T|(A3f(c;(1wYI3q&+#9q#wzyE0B@Mc?cEn%MiLEfBUm0Ea&1Y5byik>WeqJ`Y zc@wKbmirvUZvKAD`Z3@GQ!kLb=fkHd<42=r&`;v%Br#3Nagl$|A#GswhE0yvCJO7? z@EX8RtPxWpBgqWP<}hJ5_-d0jiT6mP)I4dji8?BSP|V&J@_;-vnx9UNIqAg|(w(l6 zp{!Elr3Ryn$HQPjfS|Z1yxWt_uHZvF!}fMs%4dGFZ*HBd*fdr$Lu*bIBisVqvYxX6 z)qvZ;o-cuQe0B12ii$nw?_nF18?8Pk$o>n@XX0jz@dK`jqhdGjxzG(B2KbLrY8cN% zvw!%iJgw5?`A!xA>QpYx!$Jx5LGb~MrYcG!&K0woL*F^Dx_f%|59od$FSdF89ByO$ z?s|FiQ zEseLME-fXEitW}Rt9X+-MX#N)K3MJ*uz9hzlu%^{sA?1=M3K`#&K3i#U^1)m80K%= zX^-gn=F{jh$W@gH>KYw9O`xEk*{{qm|5=kIK(lCRy&+b}(a zknK`r$IFRXEe@;AL+ZP%=QJlP*Ylpo66WARI?ANkFPLfmLNR2BEhutXfK2xR>G0v! zBHc0u&pc0CCrX{?Thb=uM48MNm2pgFWyP9UdlxKj!d<5eV(2}Zl{?#IFZ?3*{{B9d zv{WOHm5kuIUg4Yx-G`91xE@2KZ&n?q{tJ@ArqAJ#Nst!}xD8b>-T^v*Ks7X8VUaWJ zLE7Iq9DPB^-vu@#fxFJ*=7^27o2>~=@bdDS4?_oX$Cv%d<-BywOJ^_*;XR#3t!m#o z_d~$`+WLjsn_MALmrM_TXcr3NxG6_OwqDiWEPN!0jZq~#LT>_A%E~>NVYb{sx7Ph( zY0GyNdvEpBqQp5QI&S2;wntzNoI|r44 zSI*Zc#PZmZq)2niTrM0nV!|{UcmNk?x*cHpjKmA!9e!#kU>a~B2$Hlt93-?-(()aR ztOkDrb|q=ApnNd95DNle37(y$5LwBR@ZZkX@c8bwZ=i?u?mHvB7Jy-VS1oSjV}_ZS z+a921<@FU96>*tp7m2Aav%N^`E!d8l_&THM(0Opyd?fdH)mPXy>pIVS_fBAyA$L$UidalH3g@7q2A<8Bo0@8G{`mG7@-~rh!ob8(A ziRd6J%+V6+b;k=1-fP%d-WA_92Y&_-Vm9Wc$XAmP+KO##M`s2h6GLA*+qrvXC|2`< zs*I@yL)LuYiY@R7NTZ|9N?A%0!V)B8ubvnS*q=BUgCZOowqt7l8d@|Fl&L?5X%l>c zWdB$r+4HyB-iCWZ@nQwwbBcEZ<_{^UCnjgVc`pr{2%=@4ODhC=Z=w<5Go%=Df%len zza_X_nUQt2(Za$s?w3AVek?6uBLSdxNBWhQ`#4)wuU|!-IKT_{X`2oF1SWN z7n7unE6Z$MK_4|t9W=E|xgU>vw(D*Z@S^+Pe1S~SJ)@OgJrr3&P*4S|jXK{UE}L(+ zHEw?&x&-iHNwmPxf!={{-t4kLzn@JIt0pRw(7!N7-T*9D>ez0l$Q)(7#y^jTJ((oDK3jIR>=BV>)D{Oc=@X*7*!Cz9oT!rk4TG%UtEl!Ex|2Uu4venuPxT zZO;x1bP3J!1F{ESisViH=YS!E8#dkCFPm>Ffh`SW3(RFGbYQ4=Tn2 za0+!*r;j0Sy5MP^9U4YHtb~Mz=pNv`ylEg&`qz69EY43_QD_Jk@r7P+{*b`1Gz2iM7P#!^m7ur(UHFRgw=Y~?)G zY(R^aX^ohGoOTO(c1*_~-;`gNUse|*mm0|mv|(Xk#A0F8ey8;Aol^SDflL`l@_P5# zo6FVW?P5%!@}>dCo1PIe!1!E_UHB5>`d!73s8e!J!cGZY$4nIsZ{CTX&F(rNxH8* z?+Cc8B(O|Mzk0B1BCKHw49(k4TYI?v)A}6xNzxhicif-LsVR)HNucf**PD#Y*0zWr zn-PyAnP})37#Y{sVKb_0k?-0`0TRM%%L`R_HFo2&Ymx6G%%3}|VQAMgu<6#Vl{BOD z7AYse|bKTdVn1m;ht&T*S}1 z{f{9c6G@PFHy*J3@ofQKJ|6E&MLtn;PHV&=2h+Ie>`~k#SlZ9ZIDKOLfC)I)+kBNd zWT*Pzo0{P#(>=EFEFr%)J!}0ED-~XQ5Uf{m@pn5+$AA)laoBnDCqZ)*4GB;dO#9eK zn>J{x@bbs05YGFEcU;9}cSr5wX+~3Z_hbOs+Z(@5wG+rvm?$naIvlT6={-@8nEyN8 zq&jG<=^GU0*SFk$D!-oQVpNY-9I-aLcA`=fXy# zV-qrH$knAS0WHnLlQXvsb3-u7^40lBPljo0wr200D#MHx-(?JJxU&Er=V2&s8r5z3 zwoYqCpHqyC3^~~QuMQUq9zh38mWc1)Kfm6B{DhOx;!}6+T*uP_CMuaQ&V-~`+4qY4 zvizr#Vx}JjOR#rF7J5P`5%uiH~4S2iL9^Nsam~2BpnD4#}j(*F(m*V zqEk>*dQ=&Q^O-KHXsoYX8rUB2Ylj_5p?8gjuva-k0=AQOVPN+WflIW1v1&S96_Vwhkpk6k6o1+hQ(%S5|>XuNngv~3vM;l6RERHcQY+-KQN?s-*OST{( zWMhAPbPV2_wfxs7?y6h$Ps#v;Ckfejd1b9*YudB`+x|rkZ>Wj)R-sQM0KE5vHB;K7 zS^A~N`0^cA{9_-#H^!ysf+RQ?Q<0Oa|4~Y+f#FQAl8S=a;P0Fkqs24VqX29LjH*W? z+Jb^R-ZpP>*`DTzn^W2j>D7D&`E4Dt(NXo2+?(YGM}Cg*4!oYtZVB@A0J-RXA3T}; zmH9Da{QJ4hOIzfHHuCAyVO6Dkh{YVi;5M-e>B9~0OK%Sdqt%F2BcNJsuEeX>XW*k= zyQOFO1~Ga1q)U(^IXJ6!iZvTxE601wAnv#-QVr6vFyOjC^DtrDrx9c)@}!x()@#~n z;1wTQ7mw8K`M=&109eghzQoQwr34?vl32LHXLTWB#TnE+4|!bEDQ1EvB(mVO$0#z$ z>d5IU6Ui}-NbwaU)xf#8yP1O(z*y0zOZ%b-xZr{FV{!7P2k2XUG7WSzc$;c(dZVkT zsi{?*dJaRdSlJh9dl(kMIg~j@FvYm^boEX34D`(O^z;nOEHM9A)B~i*;^z0Cq*R2v zXDRe_59S*ph=?A9KS%;-GH4k_o>Qjg^=_2c>R7_S+>pz2+oa z;IAq_nk&=Qx;Wy)3VuC+-m{t}QSReX<|-bPOWAM@P@wO~ULn%25rww21k4~r9o`3s zZaKJ#hhr7vN;QRJa|&U*iOd)n36m$MPw(HDQrjj=O&S_G-?EaAF-Wbg=2kQ@qNgO- z`uz?7r85CNtgHpvc6S6KOp>uhA@$!T5ez$DlRBRS_)q1E^D?_=+ zz}Gj@9x3*G5*N!Y68!4q*g~?{V{9?UNN0jY^z|kLYX3k)tK)6qDRGPjk)e;i{c_j( zaOQkWStbSq6EojhR+fqQfWJ^2Gd#arswS7?r==d3lORhHp&0oD12PtOI9|dC`mi<= zriIH1tz=Dg0TY~9vXCI7*P~Z3(YqonikvKpz zd~9;rGvcr{MdEXuFh2}x&rSIBbx5krD}7fDt_?kNzTxVQ%-&_qsaA*7DLWBx<$}L< z{`@z6mUDJRH6q&kK-Ve;?N|GSm%o)mq;n$I#A2n#n$^~9=;i*0S#}c$BB?7?`tkd` z1UKwPzNptZ!g+YiX1UQ`F7P!9S*)$1Ex_Q+{ERQjC|64bR1|_u&jYykc{$9=(vwUTk;p+RxgbG1{ci29P~cjlh(n&@%wBvR`tWvDB6`ceHO-wV2#+ z-3Ndgf3}i7C>XDXZUj~`(Hj6v7W}^7TxvXWcog_LUjr0oK8Z;eH*Zx}m3g>HlNgh& zoatEaK9$9^3Bv>@vb$*YH+Blorz9B~gfttBq!cS3>-@qJ&eK)*VsPto+Rc&MU+7dx zGObu^TBlmDCTU(>ubo+PNgQcOS}5yL@Y>3lt&WAX*9n;SFT z>Ezxh%_a_rvp&6lm`)QVe7Fv4|6;QO`hC{lcOfp2tD0Iep75Fq6sYZ{yYKuJZM~g5 z%aJ~HH~9_h^GZje@2T82fsk;0cEb+^3Z8*#h6-huTG_n*j9fVq=*RP(ZUrmwHW&iF z`?Xar-ky5b=XwH8{7fD+qm^bZW)s&#^DR?_kJbR$yMh+E&N5iXtB`YMByU-cyjpR#h! zbKR4rMLY9EUoZRJlV3WTYdmTK~)HBihH9JeF z5C;hR^h4Xn{$WUWl9wtdg9?z{f))3=XZ&bc-NKa>Ay{1|xj0L>I@|{4y(lq5o!0y) z?0peQwgry&0uI5u8|BO`%x2ugaigLLWhOv&cB~n4ZcIGrJ#pCwXr&v;2;9!9Y*NB( zWOwwYRp<@&KVSx6kEbW_Br&MAoZmEB{;dv)Z&G4j^1CedPxe{-!DqY;+j=3|cxHwz zSamP8?s7;Hwv+GEU#kDm8;jSDW6R6`c1{47T>Xjo>x{@W5?hwR0_5C18FO-R+o|?a zqxBF0LneZ(F!at9%ty#X3JO`sKwUbHbCdUH)3XdwE7*yfEa{yOp#Ijk$>O^E7kWOY zr91_>s0(Ni^&JPm6?~PiIA$!KZFdX57Ji*Nkd(UuzG~;VNF65V+bN*8f{d%C_KgA$ zf}508R1ruNp;a?r;w1L+mW@rCQSg9;&h3wNvDN7!>)lzG!(lh(f^BBhW1}vq8ZxP9 z0mQ;HOXcA1nv&G%0VIuo`U)gl(N*kRV=600Y-Ru^^$p@$kp0*joQMKDei$1OdQseJ zvH9JnWcSF3TZdlyD$i5p#%*!>GTeG;?2$w;CccC_p#5}Is>XFAb2N6xsmjtYvk83+ zy<6D0$^{|F96NH+(bivG$x^`fjlQ+Xq7Xuzav77qE|dI30pvIr^+>xTFuxE9RO=#{ zY6F72T6*vxQbf$5a&Q(oHNy)DL@m`M)WixtF1y)}qPL!#kG+23I1>dU!(M#cr7|Mevbu;wIuCrLYQ|_qm!LU%_`Wiy@h>3!UQ5&`8@;BJ$Pts+jr)Z+Zkc0%SiIA(>rAcQ8Lvd%$%Sr*Hc?&G0SpU5^V7dO0VkoVUE~`q=NveI{X> zczENAlEUm^g4KtV2TjOaa&0$rHxp-g$d{ddF7||b_|I2(B=9zFB)$T7BJedB%U!H> zj7c>qY?qbXU@^F~`f=0wF%7UZ9X53>!4E4j+M(*drQjokvw~5xV+#!hjjj0@r;BFU zCsN?U@L%lJGsKRp8W=};D>hD6A7eOKQYI%%##ot{KfDn!8;1O-)l-Ok)x%|V)?WaX zuJJaLAFC1x%=Ja=VY;CYPZHVg&TEMbCIQn2hf;8NnwXcmIU5Tb^OrhCepN6K%0qge zBY%Z6QB#WrzJb10L==%Rox;#T9>?n`RDzF-;r8l$#c4ScF#( zA~`tO&xA;XlBnz=Nq5@3?KZH6L`cbZV=Luo7U3bP&3y`FKZyU2)lROgD#xq-7 z2poz56ZHR={!8$z{gq_z9eX#~fhWY?fNP$EQ-{KN9^1em>D>%|d0Cv&dg$^b zq@=fYt2joCYEr^GdenW5s#DWX=#(J-3FR7DId{2nqxIhPMGsqdm$`xER18$q4weWs zF0{=xJP&8z@#xc5i{XlmA&%9!9|M*Hg;FwD0J>++BR@o%C~18h9VmfJMaKR&h|7~$u9jgO$g#{3mBFJQ)1=(hr5IGaguNd^zx$l093_o12|0CABvGpl4|3iN6r`k*)kIySRr@wS5(+@vl ziil_dm;rC~LEhfp@-{?N6G>F&h`->1X&}i$cC2rpgE2#7e}C~&n-_MQ;>-M=nF}sS ze>KqImj(WO0P+rlIJuTWoFpXSog&;#0HxqZlZ;wudssH@i;f(r;t&ZtJIC^kwxAQ# z_b^sbG{M%5FmiQGYQrjvtYy82n&QhXRo;iP^V7JKVd%qO(HH*#^Qcz5N(;e4%v?8X zoUi4=Qmv9N2X$>~`E>>tp%g>+KOLms}rmlYPu z#6TB3_t6OFW_2pwQ5&*kFW!a;p6zRB3`aoHlQi>255x=%41>O@Oa4C=Kmv*RELh~4 zF68na*Rk;mf)rk~Lwz0@@u67zF>LgA7CNxLwQimttmpl!;}ExUli=nJB6QsqZntjh zp2sYRzP<8(3zpwl>o-JvtticfFKgnr{@yAMy}c8N)9cpW1YU4s z-2)iCsoxeP_~IR@AIZc>ft__u{E`6_`56xS(^Ur{nIuAXKE_Ys(xz*xh$26ltppqD9-1l0qxW5Q2ii zIzgqfvSO`vk@l(po!?WY03+>|5IH6=%J;Q%iy4XxaNC++);gZCgwdcA3ZN~fzfh;x z;ugVw9eq(GL!b4X4?jsn@AKHvPQyIGja;|Zymy%6{9s!VzEZFYQAX3h=+NtrXZ;Zg z5qx26ygq1UVsSA71x!xA>XN{uXrgXHk!sC4=E7v9G=qC2cob0L?6xoB+2j!=R(%pso%vR^o#&Jwp~w!xt@SB;{xv;MrL#8J$-PK30oy zeEQW<^Oi)I*d-)*7)~XaQ{z1P;VA1FAaJ9=glYWcEA>y>ah`A7O21Y}BzRlzH|SNh zZn@aX26fI2Ijbwb=YXR7PR3_lDM}s6^_ey$ATAgB60i zUNG0ZStn5sEV;$t+Qv4;;BA}ZlvB6z+ z?07LM>hz1AfIg=VS_e4GLFKP`vPSd<8xp`c~DhVi{=BRDU~A;#bQ3 z;->o6b5Furtr_sl|8^UBm}5hMs#h)Yy!8|A?BzSZW)S{eF$HU?G(Z1sN7m0?%-B#x!Y&rQs?@gAGKfK^zc{&DsgcJDk#9te>2S{PrWT|ylqjWl=740 zzhasM@>s|4!?LT!^g^=djOnE@HRo}TwpL+VA9YO}9Bp;QIqe(yY#R+a`200Qk&VOX zTPR>YIeBI;?sM~QY1#g~g~Y9h|5l*P^rPgl$n6rc6HI=VxMbYK+|oeQ_J@RBBt(3x zJ8{8(>-*W+Vce+Hm`wRj$M^gPX8x%L`j)~xcNZDWem+qD~mSlJ5W3G&W>+k)GM@?$n+#!rPR2z1@<*pa8Fh6^T6;RLQLN$c-T;p?- z_X!Fjd#Z|qsd9#?pc$*doAvXsR+?tIn?l=FYe?HwRjN-)pgl|NDEw-%7X(+?^p6^ zF9{$iw!deDMxKFXP1^0Jwi|TmI6^Sdi*edXGE-2nPkcVpmG(UcJmyNfeQneVzsPb{ zqhkPmC$1_<>uz1N>VC;V#?rO@kwo)}oaHgLQ(T0qlKBf?J1w`{Fh|$60$z4 zq+MLbE&xA7zK6iVXRjCIiW4y2PAv$w=QMB}WVOUbQ&3VrdwbJKcihb4>N=gdB^`>zy`!(zcbGwe=Qx@(V2$G=JD4vhA^ut)3;K3|IrLBv6Eh z1t7DfWqn3tiTK(Qa!Ht5@&b8(Hfyq+E6!DqiVcuZ>q^bln4KFqH9J>EGY_)J zJOxO1iytGTrbi$RzH^xHeLe;tK|{ISwwA~R4ZLevJj$9D%pR4C690C+`kH3xygoR? ztTDI^&wh^GAIuTI)w17(K6vi5o%_+1`1)w6$<9?1IB}{`MeDzDR4x6ECgd%|D)2H; zAu}uMnGt`3`%v?0d^CeE{6T>LG&KOOo;IJe z3Eq7|skb=EIR#&3lh>LYK$?Z%+v99sv${Mxto27--Unfc)?3iHDg={#G#bP<_X3$A2Sn25*b8JlcY z{bw7k4(k^t)CKR6Yy^-C2pD-hX9So}YPhzx`C1gFu|F+Qsa-8VI7o)&8~LQ}_aT*3 z)W=VUbCq`ERScdjfx1nvoGUEScDYuuED)jZG2qtKAdzjbkX=`{r(yI1lT;@iv@F?D zaB#c;L;Me@rn7B9E*BWsICv|3W!axN)DpPgbzMA;Uah}(Aa-KjDEf=(0sS=vJI}S|ov-Mx@ya5(x%vNd4OW zx&P4x1(@2vV8k=Vy^^`MxF_lSKB_c}uSR+HiFI(Hb&jVNLRmCPSZ`(O`)O) zBL4`vv7>aOAw~neyUOk7 zTWA?vtf^S}xVgqws=JSAs;&jX8UlZVJzc&<2 zs7?*3tb!dY5A7|)#J}Nci_@vHk%Ec5rH12`#BgjQ@%8;LIJimm8T*a1n#?$J`?6;` zJZx0V*CzhWt=qVd#N;ecoBOidaO#SPh~|ihcU}+u@Bq@R-!Be*S<`KldP^dsV{AnB!pyQOl4e}y4uF_=Aj z_OIT=Tw18f&37T4em_1j^^m=iRa17xi&9MR_!8QwJ_x;}{*QdncRg&Rr0V*r0@X0j z%}S0VkvVDEscBE)M(Sg)4k{Ih<`DnD*W#DTZ!l#OKq)!L-jw$9nq9BxS>pVk7g{9R zrfI1>rlWzOu+w@4w?6d8CR+Hy>e{{^qTi=tiZ#;8T7`>Y*8x(HGITSTkmgf zlJfNym^S1eY!)UKB4@A2__?Z{g9a;t3I<)}eDW`DOQgdn*}!WXG-OmpEUtc3)%HVZ zDxH?OB{x-G%>IbDht;-S`Xg5AfwO#Fy7HMrIs1o-)FNT#KE}PjnLjLAQB%<*JWb?1 zj={0|AIfrEVvW8u3w*C(;2zvXnP@RdHm4OA!X!R`U2d3U@945J-0IY)z7?z|b1t6O zi;PKYJcwFNh02^~%%$$g)yzKsTpw{hYmGl=O-?Ar$C|Y{p>$Shoxq_kib0-`|20YJ z$sfH~KlRW42>(PzJXz5#&U~t}oNJAGbZWjC_TQtCbr?0V(7dc28l60${jsHQgp1!| zi;a{1SS-<+k34x4>zk(&E5D6~tne%*^V9m%O62cQW#x_aZXmaD_gkFw4Cb`nXS4Tg z*A)9U_FWBs!|DXF`9yW2N7YBkK7jXcXqJH5CN|4oGM{)xi3>>Z3pCiBu&{M^1KpKT ze#2b&wfkYlV~%fv3{!~>oxdp>wKQZ;Stn{#SY`{DF$aiM<^Mk$@a;KvBSQ$V07&vV z9NXtzNsti(+bQczV=6q&?dRE-e|c+d>p?m3n0CI0`L~W+6S&GcQ?Zn4X@)t?a(-+U zn)i&K%wehXsL)t-;x%SOsV%*aBDB|Fh$L*ZPftb4yCi1*xsXWh(}-tM8utjT9WvkE zAfA|N8T{2msEC!UZ*TwP=aj)TPMyLN6wf9Zbkju?8xv4aJjeKs*6Pwts9hcAMpB1f z5*)WR+`{1RrmN~Y)7m3=bt$~~=|hyS=l*>y+Sp+1Ej28z|T&h9#b6=f4jK}e7uGTj|+Y%Ml7A|_&dmcg(46OXD3Cw4@P z${IJs{f|-0f1~t?=jERAoi%>AyMbcL<9SvS`OXSQids|W2b>w?ILfb={@_I zM{6a2xV#wOoHn1)UG-7goGm4fTq}df9J9$n5A~ewU0F1+#=YkMHwuSUL*o`|XBdig z|1faKF&cjjiCjZ!ST3%o{rd`}*39-}Q*U90MmuGy_?@e%M?_f800B^rb@*e=Lm%!U zh6ck?)-1V~gd8l2V?o1`k3+Bw&v)@C|3+kyGR0`yMk?)}$>HG->b`%4pB^pmH6$gR ze$|RiE?0OoBd;!_0vrF6{6X4niMZZ2L^&`S>+XG%C%LN-bYDeiMxvBnK8_rp>_*>@ z`Hy|KdFm_2rGM4qEMSi{kP7}vJxukV=P%f?rLQqJ*VflT8u};-XGuzde1ZMMjU3An zhp@MBI@lCzp_R&P{~5NheH)?b)7FH{co$_p;fO!aOmM@z3(Fgn&Bqy@fPA}G4p-`B zO_D6V%&W36f(=hjJh!dU3;iFbbR)K!BV?$N$eKLeVT)5ZSw*D2NM{0n{yc-$E!MQ2FiEQt_RxXs*K z%YVYeOc6j^UvY2NEnE7gwLpI)Jk7QIUr}h=Fndz5=m#@`El-je$|RL1ZU&5)ayxQ{ z7{m3pw@l;&-^g(U6BIs{wl{m!Y?2?hsJHnKTlb9$%5NcmrQq#NM zm&xJcSxQt|g(HG#EOw&bwj7w+z5Wy?I`uzPasE#J0A$XE%!xf-=oQfT;T zxg}$EAn0#$C+3g|k0&!Y(LbZhgNIsRpQ%(@9-`#-2?!$Iisn;~{%3#^uiQUm$lEh` z%4}q}HD!X$)5)4zNitECAxC6jWS0JgN1G!64xv|5M!Es+CrEw`n)csWR&M6fJI3%f@e0sg6V&+pEK5EgB4`C4po6wl z*X>#_On~VywevW?rVn@156ToZ+~EViG7}MwP?7jvo6i-OP-?I+$YNxjNd#h@OxPCg`?DL-6k5fLYE0$ zw6LR*tjqyz2ejp9>>lYggba(OS|0;V6s3zaPe|)q^DIg&+8kHC& zAzcCjq9D@Ul8Q8JAW|ZNpoEkGqSDw0%s{kW$;kDdGRs>LQtkOD^u%oKEe@ ziMKA{q6YjvmZh&H$655?4jxNGqW530;e0$-lK?cHyrQcQ2VD_p3mU{@h%Md=^6Zb8tAR{VT? zYl9b|fF3A*-xuS5D!tpAAJGz)zfYnPZ@;?M4-K^F`C7|CnZzdJ-t}?ymk~5-^B5>) zKR##34lg4X!#&9U1vxnSMCwO+Rky*Ovii~Ek3R|nX;gHlE&;miNiB#)(kyOQCXA&U z&F>lVJVCk_VYPv{qOu>3+Y1aYb@=mS#;~$lKn)e>x-Ksqjj9=V$q$gyOQ8lAvo+% znI3Dwz!v!~+Vs05Q>Rdf7F%(tBZkNgdVdmJyPIxvNP{>p%+5SodF+|n5M;Fzv_sG3 zm_p(D;86i#Q_c?DJ@Gt_LAOCiSRcQe6O9IY%n(4L6l z(4`Si*h@SeRQXUWR*bd%dz#x^uG%xuRL{%Re&`zKhn4?Qrli~Mblliy=OIT=ynm?e z=GNOD7I~?_is#q;)`w9f{d*N1W4wA6v6Dcauho+O(U%Kq z2%Zl*R1G;w!zo}yX;^eAEa~%&s;~-SRT&Yjj*wIQ>eT4EoGr=EOf1lZx_Hj4#WWU& zJ6u)-Ml97G#WA?MyPppZA|jdti#u7fJNnb04K2IHF(PK=aPU03F>t%#n=YJqj0QxC z|5@_AboO_$Je$J_>vj(`YJ>kvTr;(;&+-1EBinA_|HUUA9sDPqOEKBVHC8ZymCyBU zN==t@lvgA>ZMw)4-c7A65rIeW+R5k ziEn)XBMm z4=(5FhJ}Ys=o(_76|z~kG!5a{@B|KwuxqYQkOe#m5Fdi^zrM4+!R~xHTYQRpB(az; zIl3&O_%ZDURQCPqq>--qR>4#L5A*q)=M ztfZ*A4Xhy1ulB;})G?@3rA; zk9e`fB)VB?R|HW2)HTW0I^Emcx4D}POndi100}}r`(4J>Jjx+!zqQ3%x~6IkIp1T2 z^CbtZ%`6YHhz>5K4Ytp%vr|xAzVv)Up>-U@z0(H`+50-F|2TvB+VrEzz^(a8{j-rj ztVKG#TJ>Y26lpzexD`bjD7tCdqVNtt%gPk$PgG{cW--S1}T z3krTtul7|wdu(zKvnF>N#!l0AoZ_~0-mkL>Q#$H~PE1c(pQ$Fh`C0(w+eq39g_eXy z%;iP=PmtTl)Nc)46XMEfT8@Cb2rwU^iNbDobS`k?<5jmLdFXp=Qc@f=^u!{$7v_6> zo&+BnhBw1V_skn>lvIV9k=$%QAP#5RO*)QX|kILhjOaVMkhmeb(t z#zA@(W}TODr`uB@?F!05Um2KCa$bw5?w{*iCMNqW4%=^$1PpHUh(^CWCr8EqH1#v` zGN9QXnHPLY=lUu!0ll6(Est~q-oQFf{cL%$vNASM^RB^m;f|fW>w{6BT&q8uuM$TI z#5dXf(-%XBoz2xtA)Ukmd!ZM@tIPl0H zf_0m!vsrHQgO1kwtfODQ-)X~d@7SrJ2Z=?$6nDMChhD`D`Uko(a7ov^$7WCYn4g{? zdB4OlbBmf+x>HE1^({2}!WScG&22?+E0sJI-l~KN=u||uI*(le=#EM#?ni)Uda0$1 z+j2P`(ILK|*u1V9bO7j4w3_xcr3r~PC8D(arxza!dBN`IZzWB_uCV3mA@-<`oJ}+L zE`JiUlX&nnta)`4(f_0L#?HiM&bOK1$5bU0kjadES6^GzZRAdKUWCtS`Ux$iWVL^| z1sCMAHV|4F-V>Mi>(|!#%Uv;!Q=bv@lgVSfpn19ApMpGEJzd_GQUv?cGNO)Dp9-07 zGRwD-MRrSW%||n zZ!SP~J{!=(W80IdqG~5Ten{xv8nIZy@k%7})iyCZO|rr1$rvukR*bYo zI$|FR$FzL^uArnOBc|ZcnZ7VM?Snh0rhqmRikN!opY9Ze71hw+qz;KGoW&e92mHBG zgOpFFI4NTa$*nKrFfr10XY~y0 zOObR67X5%IlEKGDAvnU`LP$7gcb41%e7wo^?%0+}cwbyNX?V7AE1L6UL?ire;AjPNUIPnVht>1M&=EyOqMN`ZOJBD8Tn6 zUsywux@Ih$!guJ_XyR03yXFN80UqVsIFoxi1FsnA>$jbNCEeF*}^dVDJ; zER1qn*FRan5dsOx+~`r6ZBa@!2f(({HNnKoy(!eZH0Bbx>A8`yM_% ztxGNVw+r{$DfV8un~^=(KC3kzmrZFY|9}!A|LQ#dSX8GY!rw`!{_B8 zBi+ACO5V^P^~@!~ac5Mv<_(T(zoL#n%?&&tm57yZW*Mase@<@ax1m%0u;pTQo^v6K zL(~x>11!q8Be`i`)D#r|WTGNU6^}RTSPKz2L@Wj^;dSy7w5q@d!=Rx|nGbzV8->M- z_)Q@G42qxDWCC%rjJ}Ck1wLJDKVtw!c&^qF+qwkK;>y84P@l*YF{v`k`yAFHtLeFV zgh67?Iyi!pPaW-tuM#fEIx-VSxga%)8 zg8E2T!xhr{r)0XSWDahcV#PS>gWEqkL@Q!U!x)tC?W?UKp>RJ~Qi96QwxI>j63_Eq z2;qM6qH=fg=K2DD54K7XqLci{7mTqHO4&4QcD_t9$STgz3m=<7j?A4LV$b_%bPB@$ zyww#DQ|5YkTvOvNAi|jB>g+1f9?Pm!2lxn@e`=Kvjt~HYlyM(MZA})dy{1oXOv8#( zSAY`FdDNQg+NR&1L4po(q+&V?>AVs%)h?M!`_4PzjQKVl>EUeRl^kIILs&w=_dke* zmb^?fE!Whnui|g?kP$*CxePU@T3A@rpTgD|%Y;;KuVh$+;zyb(Utia^y&hSj>opJ{ z$I$97)s7GiWv3063Ei-km;!|;a6&xwSKkB_`y-K5$$3j59$9a{W#n~ujS>5cf2kbO z6;)Q&40TA%55LlN1jUwCwY_JAW~9p=^^CHvepsgNMYW#8ZyStF>AG584 z`faFaJ0-#ZhhiWC-V!mIHY$pRs-kS*i^?h?-Ou52fNCP18#i8gOxr0)f5@Up*$gr1 zO+Fteot8Mj77q&Ii$U3!5A@4R@4X_tFZ6vO;k%)ge3K=~*~R6myt0@wOM({v+maH4 z1cICq_Iu}n*z|C2$f0>8K$H8cA!o#&V`^bbn4M+SotyZ#n+)Vm!#zSSjHhaHn51gq%}Bk<>HI1yjwyJFtshfF;#_#s zxV}3e^@RH71+R2Yo1P_djJG1dr^g64_9y1^AJbA)?8v@!US5p`pxEz2QNiRr2LwBhE!Nm{X1Err;?nRhYWK7t*^w@MeL; zfX)t#RP4vvy{@ijE78uPaco-l7Q^E5RqrV=?{!|G=x*OO-~`?JvkS!@;NswLxlL)i zTl=#*;m}k=B>Rm$vst&k0|Cg^dB43cV(NsEFf(hGvgvK!*7yAR`Jt~xZOfNZ@0oWN z@A8BW6Mgl+Btk1s-TiI6eZYF%DMx9iM&-PlQ4b>QA;#Fl&yGSe6BEzZaq!#-t9ftS z?f~eL=z7M|6-TI(7X(_tYmr~-8-GQnp`;D%wgprOrR`cTjLF5Rv<`wp5V>5j2o$PQ z4zg#bLP!b&V_`*(Z~Myed)e1o2bd!GMloH5CTUoY)n3Nn@wzkO$c_X_*RGUT-0QHj zU3gm<6iA5DoXoDwRXVSV^35c|=DQZGmq5g&BjM123noNlfOFN&lIQaytULmLCe zzBTjAP$ebn4H}8xOaJa%*L*2mP%AGYl$6Kc(w-wibDxSY-IkvOKL-tkerk(iS2(o3 zcbpGi`xAlKgDy{T+@d^v50}Jfnv)Bh^V}_8Y`ueVQ66Obrueuwl1g0k=5<-uE_ChA zbh(Y{6Ri8y$D%gWKi4eMIuD!6)v-DZTok-eiE{L>w>{{uUx~jxvFMA}9$fR8^sk9u zO9bV9JaHg81+i<@7LXyZ0|)mGS0ueG(K*sXQC&d)N>tkU341jca{yC+iF;aA>(o6B z0juisLw*X}*Pc5Pj)yG*5ZH%nzO7j7lH_XdaO=RC-hR>z)E&D}8NpU`WXc-r`!mOt zuT7TyBDU62erpX}zU7ORJ|`(O!uR|*;O1KGeI6O-P&>@g?5)KfyRh>sWV0xWT4Jkx z-xA*^C8~=L;A0Qq^X3p|xDTM0V}}SL8R7c|!*l^!HbI0SPoSm;&A~jkIR;-{R$BVe zXaYWLR&8l;pG+CJoIsd#@g;2t-{|52S80oLLM6iwbm^j$Kj((gPu!N7nv|QAlbxQH z^O-x%YNk4wCOt8jEZyLfY0&z&~K^-GDim&K7^sq0~>UX&joFroHH=7r5c#FU_ zd!Ec)N>bsrnWyN33OmcVb?Ykc^~8;hO~Ji)X`vT~(c6eWy}zBuI&t9Mkw(W&Sg~ANn_pi{T4qCK zS>RbuKw-qKK{Nw0k)~h>bE3f1DZ^PzdC-x!r;0%q{k(lgI)vNhuG4QUDXJQIX7pIJ zXNjq_w;72?w5h;OA4*jPIs4OrB-NN{U!nPb?9+SJ)kvptJ>}CSR?^KScvT5ja>SV0 z;TX<1u73bH>+tD;ZA=F15qQ()Kj#oA7F>=r-dwHD&%vM+_wjLI;FDcnM3VNG{a2*S zEM0$`!GWU2R`8hzaC6H?fP-~uVU>feYLJ)(?mbK;>uy)ONTOL$m`8>I2SYtI%u##! zeCPUgA}?cmzQEcxDADnhjw#ap@m;-#D#H8uIqvVGR>g$B(I6lvCL=~lDskR+Yr(_5 zJ}O{>h&}0a%5&a6-FpG&^Xd8Y>u)Ka*cEGEQ;+>7>v*#Y04VA8 zsE63S6!JEj48rY^2^3nC2$^9KBg|jCvl2c zU|6arpt5yEcMxruTI}cW_Q*_H%WNw}jJNO|fR>}AMR(U1&~+Pg>+_KxK_9lW9UB#h zwzDi(_~CBiO4)b*}0~dayntYigdV+NqZC>s= zTs2I}dYg%up5*lLC>iV*3r51*Hd*-qCzCptS~j{j>q;zDPMSt(xx=n@1}0isefZrk z$Hm3X%g-2P(;MA;e@N`s;^`EJnBhs~?)ipXdetAc+c;woGZaJJj~k->svAh3dHG(L zLEGtu`F$UO^tAVBT*D&?UC*i6BEfN-HsorkBeJd7?q`AUQ zs(y=HiU*b?M6HJZ==Ji_2AyPwZSjp-YNil!*=O6j5yY0Gqa$xf+}*uM7_oxBEvE39 z;7;tUa@f0>(-`%SBoSfOvJpRz%EhfJ16Ku2dQc}u`uaP1xGAPRz~{H|=h~=76X7?O z=jgdfPdUTuK^G-jIi*B zv^1{KH(v`2`+vIy6Pu6RCX4T_{vC4$MStI*I)P{~-A_4!jiGfDqfJ^>N(e3#^pAi1 zEeyfi=YJm8Wk*HXOFUX+=Yh0ZIrJKn#b4qX?ICk)C*w$UdgRKVu`d~|hX$rQ7QeE9RNe{ z7G@+QB>Ys->!H0&H>dmE)iq>D?=t%Be6Xgsp}%H??bGjw!H@nX3zf04mh5Yef#Wif z=r6ztQ{a;_RiUpr@bAScW3#^Ufv=yH_dy+WP(wf)QHYaFi@7Bb}D z$<{nf6IKPrt&WO_lS#(+41J*4!32=qZ-Q7}-=5Kw5l_o*+PYv+)}ZtkJ!92JK{L@< zu##?2=K7TASn}zvNr(9di%wZ)NWr7fxiTi|y8P`YJEZU0(V8vjJh}KU))1UXB}Fd7sup z2MOuFE+L+;(9dnz6z9gSIkt;BbaO7ac5TfJSezZc9AC9Us^GmdP^^sBn^Uc&8>U^` z+#srX7e-wQlMOk{E=|dWZmCFV5TBv!_pesXd^Da*%oOBpelUkfp_3b1F_t0|{ZnXU z1c*+EPf9Ws|M;$xG*FdUawe^poKdu&+kWlilxt5UT_i27amOfVsC>@KG~!$rPJ#<3 zjG_F+Tc6RxI?CIj7^^ccSj}EpJ{bB;7b1IlMkc}Z7-SK0l5~`YHT3nR{8d!tE01T- zFvN^Ajh36A_w+sG4{B@nT+`-XMLi{C&^&N`EW}h_Uu!}1xN8eia6dd*{-Q$0;R$Wk7oHTv0a+_goD1_Y^b~ko~};HIrT#jV&f#~ z_GBZ4wgaJ+&XeTbN%8S2iA3P(`XCDy+&N}Onapli>plKalBw#5=I3FN+g+09HT5w> z@Z3DSw1!aHfMqrtQ6+jSVw9yWEU4Wf!b2 z&p~ySl-k$QR>N>re4dN;%*dI2tBuo#?ZQ84DlddFqvjec>w43I7vr8 zD94bHxZno(OVQpg&VSN29U^iUpi9OjDtyiitL5p>$)f9*r{Or=om%5Nh`1ci$Buqy zL@COpskai249_>|28MU=02J%@?{LArvzPSU(Dsv`R8^V)pKfX<5~_kn%UUaUXJ!&u zV^=|<;@4#Z&)<|PvM5zWcBHu(>Ag~g@+7mzEvj<9QB~&m-_G%vUpzbjar=Dpy@bd= ztvihf`;odLCqe9fMdolo5q+UY+*i@#PVE9~*0rS|_CCIPtNDadNl}9=vLA=5cwNc3 z8!ip?in5GyIfKA!SRvsVxRByfS zDOH`XEp{zB)}!lKLGN&rnVHe=7)Q2zK0Yq3Gm_(J&`ZOBEoiKiZGjksVLr7Y1iKkF zTCHTIzO7)d{CtmXMB`a4*8q-L?KGR>ZcMF@d__QM3$^Tp6nj|(+a?M_i zD0kpM$r{F U`V)F$2b;Ok90+qN^DrUL{PhQU|)QAEYk@9tp83qm}g!R{T4II0L} z*{45kI{6f<0}5$B55wM_CJcVflmC6XYNv|AfBKSP0L;kSh*<@R_whfN?BM6(65=ET zm<<7r%E5%)vQ~ufUcQ*(S-_$y5fap&A9hj&U9gXB)L2*FP(;`bs`j4%#y1SnF@9pY z6*;?(Rsx%E@5Au7R|+eWHJz<%gWTxik!EUdus5(hds}s&fh@| z(7#^0>zXh5mMfc=l!LPWHc%?r@LA2E{59-RZ2qWuh4=XBIMU|B2%fIqhOp~2T0*qE z*HZ@EMKP`3&4FGE%U2hIcaqpG{vZ=ia27|dKEeK-9OYu8BmLaUU}j!kJTEJ)A;0f% z)nrc%D+}wjj122L1V+*bWc74wqt|dxy)5{s&XeF!ZvwWlpyi_@Os|Ho=4-ARJgw*9`|oUro1B51f|0TKSfp%KnNZzrv%NdsSDA zDa`YYe<9?&j#G<(i_HO@9_Xpn_6&zo8cO5qc@vtP@ zXQ68Qof8SNvex+J>tT1(^ixT6cth5|Jk(lWP;As_G+yg2Ex z@%p{FFzk7PyTJIFPM+Mswt`7sn9Hmoafai-%ipbG%+<>gRPwRMWs_jOO_f_(!qpweJBN z!#SDEkPwfD$yJ}DftL? zKA?FbBGq@dPGCL~_UH)9${H3lG60XKCIjkp=#J^^rRNve&A&bPSlVItrKC+b zA&4X2-uXwQd!SmH7dw@7n(~b6%}!drXGfRW>>EMM@w`H7?77bl=o&x1!p`xmV{OUe z?pFK5;gzMI>h2VM+~N<12GU}9u3Sq$%kx787!l5?ILt)T&1vop3R^$jJKphq`ibh0 zfn8dq{98d8x{rc&RClm%sp~v3^s#jyY%eiC*!^gGxD^9giy9V2LhG+|t-L-jF1CcGay!>nFX&ReNC{YyJjj&ZYunWgnyWZ!q0+R>{ zxw-}$Q&)HgChYo~6)1Yc>?pnBvaCgUBI$O!j87&B?yAz#tDhfkwpIp?zpCYMGvwwt zMPQLdl#fqCu~{*R8Koz_gVw*l9Dnt6wNrMxb;$wRQgA?Q)MaM^tMHmsB$lF+`h%6i zPTy6+QQL0M9Df&_AFUp;r`?5Cc^SIEZCPo{3|^^<@$cnB@u)BeYKw?JIj`CGT#3Y( zEywze)3z`?zEZUfJ0ndqT?UUmu~H}3xG!_ZX^iN(s`go(aykjvWcWjr*6_apIVeTG zm&ln7BUFuWU9BSY*UFF0FLk2ro}E(xP5vuN;zp^e;9F-NqJiha+Q( z@Xz+#R3r)GX1RsFcZPoR-yR+wRt`Ps8yL7_icLC63R(v{XWHo;O z{wH|wFelhYCiU7IxSx)9fcHY>9guSA#|9qmQ9WeRvZBh_$=rt~AZ}l$yHNWAbu^Iv zSpH0o z3KzkHcWOL+Rl=b7t+!vB=2})#cjvJt-9%Rt;i!Cmes1EpFdti25Cl&{+xxeC`hF5@ zA_@XYOJ9Ejukau?rn~bvE&c6Vs|TEeiNYwaA)~hEPUDpypqOr_5W&I=(o9sL+eQ>t zh|yz%!-~d36|;8ql*=?JOg3we8AwXY#zX4=ZR3C=thY0-$=Rb*k0@uy?TB#Vf)i%7 z`9}%e3vl`G@RkAyZM0o({i`D?pUX#G3{7b9`hbT4iyQgksfb&6K=zE z(@>ch;217Fp)>q#E|8a&%M7v69PjAV$!e5PwYAPfTopvYG06mQz@HBY0?ymt?G0k( z!u5o-+Nn1!Jy)pCQ1$acO&}=5x^LR@J@^?^q-#$`hf}o(y>NRIecuF;)4g#&&4kRK zHz;;yZjVd3(Vl7vti@()Q3mQ$W;$zJZ(mdY5JDNv-h&#;cpkzwqvr;=%Jeq(dfE?s z$)MZA>tYe>0kYxgU`tC&y%0V4KpVRSJCKp^qL-ycMwS66-4X=PD?;{b-uDT7>hVw@ z^*u9G$5vfWozQ_pKPJWmHv&)LmW_3F=$Xtzr+Wjhn(%l8iwfCp{)+vJBk%7*a@?1% zM0;1p44MH_&Lzrn5=X^_Ssi|$SH)XD^o4EKs!C^9pRL9SD&L~&SwRu+oLa7h)X)dSiDRaiyN=a-5~y;mZXrsa0iA zLUZM)6jSg08p^2t$;+a}%l99l^=Sce-v+dtt!&cWU1DWB)7oUB0mun6!ZA+I3A8m{ zj4h?JI^~joZmcS-C{caL|XZ#{3M8aEk_kcRtQ zQSQC%7eT6-LrJ_>(!!lrqt>e|HshoWuR_vq#BKz4fKBl`NF;M1!%~;AVwc8HenF ze#`5o6u0~(m(9onU zXNJ$YnqYG%XQ#-Dp&BCACbQ+qt;UW9B;w$P5h>q)<`!V zh3F+r{Pyv|OuWxxXSab6!$FR1&GfyaTC##KGlWDhfhYtxrgIvUaQF<|zi$8UTf##F zN|T9ubv|sjBl{q3v3tEFUC#p3y!d@cUcByC(+~`_;(!T0AC=@y?|NphK7{-ja2l;g zM@Xx;&1V`#t$u-cJ@HToIs}D2J*oUEA3q?)!Cl`ioZTI;fWUI zmHq?}c%_3$%y3g2U(eF!N09TeK^}K1grMZ+L(IS)qyCkUsW$W`KPJ%b`5)|<3faXy z%G$)Bmui#C+=eK5X3!+gnC>zgVc3d(K!<%*XL%%-Ek&kRuT{j!=r8E=?eogcxE3@z z8{rdynD2&$Av$*24vc~b$ICtG>E-jG3q$>^d5O5A#k{lph>+Qj1FwvRa;I!gq3D}y zooCS**|@BVgGV4rl>&k`r|su|!@cW8CK|4}sqy*=h%(6x61Lf~yL|>_Ogbqnd6%4b zBdXTf08-*2M&F-`r&&=u&hV4n>r?5*=IriT`9Lqm3czH&*p_%n90~{df`ttkoAgK|Q?~?kzV*uz_ck>e7D|V>$N`;d2za5^r%orZ2m7wUwYq zy6a3DYtvz`z$a(V@U99hAbj=;ib8)~PdxJqbvuYj8+nNBRt_`z0Eu`w`ie$qIKInq z(~-dDM87v1g75kxts0PRILmbksXy+Zb)9kamUc|HmAra&_i?K66DlhYi5CrknNe=Z z<|Z3aeKU3neb`hpBe3}ry-KM?gRETP=!Yfrp#n6TK(nNM)tK<`mK&JZFK&E`= z@3{QzvntEZ-UzWD|7%58qIVx&CmG|>t@GnKfc_#p^PKpLk6&9K1x_jOtVqV?}@~?QMTI;Ua9ECs8w7ela6|={PR>tpW3HE=6i4&=gTxM z7_U&E{zU|J{P?IiPh=+4?yK9YQK(WmV!aH@ID2Ruf;u7V$Sc(KnkA7?3gS0Ph+d;j zumU0e)|`WGLu-97QF35cwUFqIlT5U-K#i zWbSw0gB!FVmwXgRB>M6&JQ8af!F&p%>ak1H49;=pe@jrrGZjp)FY+?;m60ZbOEPEY z&fotxQ*{X+E###yiekEh;v`R>sep~-bIb_DD$A#cf8-Rd%xUw<2f^ty4vUk zc&1uJ_kbvp&myvO)a(^#8N!WZ9gz`r>vF|mrEOX#L3v3E}wrZ$K2NRudK3RtDUe-9*{uGda z1Htg}_y3B0N1T}S2L%EAmD=09>z$f&aD}3;>2ycz=D@%{pS~`oTQ4xyn)$UQT8TpTW>R~IZwloG#S6pJlP4v z2rA@jTl06Z-2}>KbYfmdPr^FNt_B29kZGliGj*+fut}ACER^zAprZ034`jO-emg>s zCJdmCVGh@1?HTh) zf{6o}?I}BeQ4V@^LSeO!mSE(m6c?npZNXr3Qf$skv&;8|rXADZ@1(*E7%BP+orK7$ zj=)M%#%OOwls&EaUkiy7{{x$#%;ThPV&(lKSiJ6j1wWm<59v>2TKxBnYKM=C>(1Lw z#Y)vHNC7B?zX}7^=AKrW4AK&w^A7y^+m8S z0yrgnGj2^af}FN*SfOx2iYeMYUiT|Yg~$SVmb1NAKg+!yCu=P21v(5OT+P7Bjjpq zRGzBk0Dg)Zh9iKpuiq9j+_52ACdI#N-NOd9O!YcRMHTavNV$8FP>nQZ7% zz8Fc|Lf|=%#9LhB@T9hv ziUm+cS-AQzHY+~3k~v%bt{_STHAE}Rll999W>=+IQkQ!1r9_f##@%M{SE=G6_WkJP zF~nSrZ~q49H^N40&FWLH-A|n>O4UlG%;O=*^V)`~*_#t5By!6uEieOU6A-cXS?WT5 zUI^n0Ae^cLtRCKdpg}bZmZGV&6YMkpzQ86cNhbP~k^D4M_in%i?~pzaB*EnO2iio5fCdT^>^F0Bak}E5-G*f$BV()=k#p+jl2cS{&Qo;n9a-TRI@pJW{ z4UtF&%+Qp-RmH^B!26j70$1!7ovVt>jaLd~28qKF?NaG9Dg9=8eh}}G{}!~K!PfN* zcz^ZP4=%*J>~&I})V7+v11{LB90G+m*cEsh#mD|E%R`92cUd*EjMg+K|K} zDbZ!!7voV^QNmvh&T5#aeI%yzopST$lQm?s$V5AR33QYm4ewCJUOIrl{^ zgfN2xy&#DFd!{C_7iCYPh4)dB>J`I^RH!Bc(picw0Z$j7%VcYnQ0fqaid9hHTP5n0 zEM=pnH?}FQ;hM`p4dUV!J(#VgnlaYuFXba;`@4LJ|Dt^9O=bUk@5|5lXA~#|4)qKE zk_+bt4^u>)6}gx>Z*>}%6xz55_2o|QN_e_Ekq$;bOW2d)3oa}DJEjlaLE#UrejOnd zojMXV;+`?4K>(Mg?@HdEB4}K>V22lHujm7UqM~LA94D4DMBmsDt7tBm)UvrRDmye%qNe0%7ZS$QFl8b6?0Bjzt-pY7DhHBWoKI?%KHG_VWyU0G3)?1M_b@Nm8BRVkHKJX;kGnGP6 zjc2s>mjKa)_Rr~}3ORMyip>4PfExHjpvSP`KpwIGF0>%-IjP!E-KVb)^1QMGUz$I-DBY7>0Nn;!9jfZhvR5&@v69ZxJa?(cy1Kfwx>Q6g^sc)S8#*8I>g;uq_ucwNB);g6K8u!t zgfs1+jo<935ub$pcc7A^+_h})+>YJZ?P$R2HY0zVC${jR+r{NX;jtd>BmAI(T!i-H5%GP!<|X&2H282q*a~K;YZ(%m*?za_X_vi3!MHFL z>DVNH19j0eh1l1>uXK)>rVR;*3Z)`0z6TBu`{q1;3KpIlF9E&eyw^-BsF5NvU3xhn*CDdg|vRHafWyhfX2CA*i; z6l!uB@EBKaUp?~uA5sbB3k(aXqiwLexmf3YE64V_6Hfe?2(b-c%_Z{m(IG&!a6bi) z#LNkyU{hxZ&=JY&xm-#0$2zqzVW#~=?&1h)K_%)XSv&%Wn!eW!`4ETp1As)Qv_R$! ziIF%er@i+2*?z3dUo05AH@rT%H|u7sx}>>76&7t{rh1Syt@<}i36O(vY5x}Yw+r0j zim>e*Q)14hK)ez0=`PexlfEFs8tAK^2)$xP7K}#-W|?m1tUSd!elORlpj0@$E+_iv zgZM31J`{>gtIIwtCeMb?%uixQ@FGIUgtIYSj8$fz&1$6h?HFq#2!p+j#-^N$HB*12 zO|i;6=N=NPU}C;t!sEx7tYOWqTy9k5i}@Rj1E8xj2u6<7Ykn?swIzY1DRjRk8F-*~ z4@5}olm3D7a56J}0z*#P)$&N#JYNE+{y;#4y=WJr=cgy~FQTtu4<2mj9VyJ}U*8+q z>4fF1xIMhsxW6^(|M?ezt&R1b9{BT8G=?^v5V({!f3mIa=T|522kZYg^8SL~|9pX1 z=g%DI$tvculC8A~iR+)I8?mJS!vOx)b^hD+_^fkVjXQOOt1ZGUPvt*(Wu_RqmoV?Q z<^H`B|Lc0<$=(a2R(R-cav54Vn8EazuVGwB*mZDu;y-_2lc}1T35ask+shka6?J;a z{a_75UH4BG$^U+G!c3c6n*YZ+5MD8EI&nZZ7=NV^gl|>4N&X)nIlmN4U5$qSOw}?|oLr#7uYMH)(TwDTM6A(}fmW{Ef8L7Vwu}^VHH|4L z@RK-0O;|N9Q;m`%mFwmDOZk7c_J8i(#eS;R`O0VR|ABijb;voy;}Sl9`u5^=^lLV$ zmFt!T|H$b7MIqr3mu6@0Q-9$-P019=gN&Q?CS2chf7%T z_Z63rfD6pmk4spIOGpTTK=8S{!W~?kJRyE?J_ncwxCu&lz#LtjU7Z|%ml6^X65jNkSJUB;+Ncp)Dz;x0E(+M!qFm^Bz6Lqv#HZ^on@OSZ1_eGc)>w9{L`1-0K&7DQ< z9lag&wB7EA2I?y4J0VSAz6d3Rx{!dUE8I&LqN8R6Mj-DeVJPe^dV~dK>BZ8jB!(UA+-vu67E3X5cM9V{L@0YoM>Ugh7C+kD<34SfL8u zCc1$_2o)u9Mcn`&gomlN7(~(CP1*mBgp#ob)DBecBCMe;?1zvCmr4er21p4L15E)n zBbb_T-?J)$P58d0Yyzslm%2xjRG7U)x|^=j5Smpe3bRX zolRZs;qIDn1x+2On!2fgsHrwoTi3+NJW$P9Pz$N6E2i!!VlQUrY%bykcT^Cw)7294 z^H6uwl@t@za`964Qn{m~sUaw!>8q({>a3^XX&~tZF){GhyCdKyq%UeO#Hc6L-d4|U;(heKvNF`V|fn;Z@8hJKf+%^*+|z-Qpi-@Sx-<&+#R$f z4^?vrbn#b!C<%%NsKSj+b)X)Oq7L5Xf#w1bZ)I_#052a0M=fUuI4salK*PvUK)~2d z3#R5Hu8wf>hpV`In|iwGh#|n9SFp3!LU_opt?`B-8|8cg(dw z0WT9LZ@8BOtz8v6BPSOJn5UDSoj{n)=5*doJ!#SCTgju%5>qZZI;~ z`Lx8Lr`~h>{tZprSZ>2&1R>3NjgCMnQ*dnbrI1}!EJPtT8RFIHi_ceYl#Z@s&}hwjkD zu>brK)=Bs2r0K{_ZSVhDbPt_MSab5xy?ghaxV`5;eq7|C+lN)R%hUa@CHL&5gO>cq zx_-A5o_*+qDR)Gg!pZ;AG$?)uU3=ia^$-+`yL{U=?Tr5s<^Mbu+S7|=Q!M|z-4kk$ z_JqUrzw(^V>BuxIm)F}X zskl9EF}v(jc0sew>ES%rc2}|8w(EA`krSqQZr#~ljfc<%Z+!Dvc0E6#1QbC#{qDVR zaRAr?gJRVSx*T-SgMO!sly;4E3z!sNvws=L>|~(B3rP>?=M-%h!n9vK+HDrYMF)5F z_hMKWJ!pjOi+X{vGQK%belW$L_)-7CUmXd5uz$EKm~T*U6o2{Zk3F=d7=Gr=&qj>k zJE{HB53gU^Wt(03fIXUGx-HVlT^6kp05<0OAQ1)p!_PU#ChA6EsEvSzxw=@ z=eDg!UbRNs-qnR5i6DXkvXT1m5K7J8QUa*PFLN z+u@>#D(}J5lh3?=O~7AbH#|uzdeMF_9pOQ%#IE^0al6?WxECpJak>}6t5^eL#b{%w z-QTnkT=RrDWZm?&miLO*GwBhHPj(JhLf_uKM27bs--p$nz@w|Esu?4GOiQZ6GiLw>Y|fxM(FUetO?FmbB4q117*_pdf2PEzuNYC&Htqb+fCH5e z3o;uaQF#Z$s~S;E8m?gB%;apprqB(&;A0+3(yshCz*nzN(R|^gVo||m0<=x-cKo6z z*f$6=f3f1O4K)f@;=+CF%36kf*tc&($9C>%a`YOAH7q?EdTpGQ+uJ}z=~wrD&|i;# z%4V{W(_@_W5It^)r1Y%PZ7!_Gg!sACHU{b@A}A<#K7Oz}^eL z)KZ~NSUJncen*xB)Ck9|w`hhhND$on?g}XJB^D@j_bW zo-g;*iI#6>Y5B~2)A{(Z0=v5LoG;;VAyx#c%_QBJEgrD8f9q$OxKqGLN=FQTW8p(T z4q>B5X(1|V5G!KfPR$PYhdb$)xTpAS*T`v2l`T&-U3)U2Ukwu3C4)_!V`lh^(owxc z^E7Gkl0^9$aZ#rr4dJ*{Jr3+fhr!%*V@ihgPLdBd^g>KtT)>9UpuoE=vI`Zh8P!wH9b7i zc~|z@VB*p0DfIJ!ZaG)%mBHARIq#5&>&|}GEB;p5wwcf90|T%^q0>1|V9-(J(3MSR zd|Xb~^T&lY*9TZd;_3#iIaZbTLv3&akO!kZ)R2vduX@O!jlSh~4M|hMHSh&)QYmzH zA%z1&{rHD4u{c=%PO7(YI^N<&>StY2_pRt8(n!rz182RM$J0`|&G*4$G6$xO(DrgS zNGa71)}5kSB14|D5i#7v#_~}j9Bw?jGWo7BQEg)`v8|TL`03XEqf@io3wX(_1MNjR zosVads~1OqbglJze?5y$-CEzG>!xp0s+aw?Zk`BcXS+yND3dHZCB0ScN$n*6_+H@9N%V9XQ66SsVO|o= z!!KmOc0M<}toCS;o(ftd`Vu6D~x zaSc`$V3$SCH#Uy!!HlQE8dooUFaMO+enY2=)U@wJ;o@6snFl5=Qx=i88U1dJG%2&Q zN`8^~Osa`U{%A%2WFD=F3?;R_?8F`%VQ!54EQ+XHVohE@faR6;ON49->!qhl_5@io z>>TVey(Y>c76&5-^BElQ>okH<8?A4_OFR?u_?C_vRfTazhCTx?w8{i9LIQ)S?r>y4 zoMFD6bU&B9b>U|umCH;bBwPH;5xDNlZeR4%Nrf`lh2{`C&amBg zZX24mI$=Hey^<*$?)6#e0G4H|u{t39X0pij`5@&h{Eh*1x2fzG1KwZ;3mYk`tz&8P9MKmBQ06+K8clVt50txtvzU{^zu)~i$igG z!;H*a8%KYzyQV$>W+}Eg{gE|xUS(Fwt^AB!mCG0CRTc~2gUa1U`bUeFZM%;|3PA%x zh2M#$YT2M?k|oU^r5g*S;wH)&!6biF+K;8swGE%`ev#NH!x`T)`^AeMN^s|7Ilp#Z z8kZU}{Bm(Dk$aI@YPiGpRG0tHPbeW@7g#HDV$#cBd~;u&i76p>b9SD&IPiG<#LDE+ zKu^&iyG6|;>A?J)HlbO=4W6?JP zI}>E`{61d?D<3D6_t`f_`}hXg?M*mbA{6h!1)KkBWlj#dI5SWGkm5-uBOXTcF_L8E zZ5MWyFP9zPc06Qy?WvVDfSjDlt6KgDLWPXV56AR2`9?%oP zP$`dtYo?D5k;V@K`*WbShr{y zsV8fBtZgz*)-@iH?Vf+|?l${Oky%B(^qod%ibhYLMlz2X{79=J=0g0==6B(t?d_mf zoNdy_n7JsngZOh$mvb|L%P8Hk<#f+-Ux( z+PkM%`cXP^lZL=A%}Z*c-n~ActO!N9z8b=j+d9(%A8Ip{a4uA7hbufYy``~jDVw|5 zG;X8|Dk*1ksf!E1CG1NBZ<;98*f@rB&mv3Cng~MhT-IC z=Jof>Lw$)RcaZAbkf(nEcbTgr_`^$C`;X(WJK-$MwM0T;GdDi&iM8eEIKaP z#M&X({CJg_s-VR^SB7AY?hB`S8jTRuUD>ZH97cz8Vl2nDjEN{Mau2Zvcnz_ow>IC8 zj;yi|E`6X z-W!xRcGSjWi0ow5uTl3Ej#`%l&S``WkJ}=V zQZr#vk7)hwI^olKP0)7E($$OFnaMX~X>ix1&`77vxX?K_n!Ls9JyzuBp<%cZrv0vz#IU6n;V zo=4EdS(&0|TyZ<$4+fpr;DcV;gKVpsmx>vgSaCjy$S`BdyXC!u&d0H#*edZBX50lg zD-9hpY~)aS#{MHisBW_yZTi_nQPoEo-p=WTgr@G!^zUr0CDJqI4?GS=AFEBTIS|Ei zh(K=>YH>4DO<>}7Ryb_7_Y<0mUc`0^Q(A5IptD9u;ly&uo#^j<4)xJ(arZl6q|NVl z<$?$H7}G;=Te26|7H%+z21cdW<(3Z|oH)wPY^O4U6hn=_TxgE$Q1gI}h?Q4QJ_R$ahvZfsqi$iE95WGXVYYbT%$u(^WW(ViZaeI!7f6|( z7KgW*j=_SS9d;K@?Q5=np9E30VQ!Db#(t)tGTW~P(PSg`hd;r*g{rRlkZ z|LTJdR@n~e-l}%R(>b$NY_CFiw7NrQrtj6N1Z@Vnw$`wh@}u50h~!vAUP^NId((t= zz@`3PiNvFOrC1s}l+RMFZ349dtbg{ht?n5>C>=xG!qu5|6?u85GwY$HEp|fs zG9v2@pTPg9nhvJa$WO=7tPjejj!lBXt-(;iEBr<)4-p*x@tfF_m1!!Pht{9=Q^G{P z9=e2&9LmgX9{u_8M%>Vbwn3+DmS-@>7wbTy)3b8UZlBAfQoA+5qA6S1Jc@?<bwD&Q5>ytc;kPK8U!Q7Ht9P{X8QqGV>q=U+|JX;=?$$yCOq0ti*rl zc`xMM%^sq7Xo~2!bL^-H8q1_q;^Lix zPh>faz86Y3VX8_g?%4j<NhncX{-T>1MQT?d%wPZ@FSk7~{j11v!Rd_}ouP zM#e7DfeO;PS__nrpP_pxjr0~#CR&Q(a3&|M=UetHuZ1x>qw;dt1>ouUf}Y~H)z!!^ z9o(8~95KA3qK2sUBcg7DWcgb7A3oWnYVD(?jHZ#@L07+>nNIg5_+|GUAU`Ern|X$0 zbN0foJcpbq92mn_{xoL7yT+%TGmQ*Hen)Uu&ZSLLT81|WUrG0Yg<*yj76!&p&S`Tgy$9@+3!-u=?u~^@cb}oQwucwiQ}CK+fQbj zZ8k5Vvi|eE&ui&?YW8DU&KFxPJ?XfB{Um+aYE!ZF^$pJkMHcJOid5fmWxX4 z%sF>dFIZa;Q^Mc5;JX9M5*uWmZ&LcKgq&=}iOKU6UxpGA?4T8$(qKGx^8Do?v_T@l zd)r#GH^V~p7y*&4?Hw}yL3HcfWMxqB8+q3&C0M4(Qd1-i$7~vVOVhX; zN|2LPdNJvgX(DEF6SkJzQvu_(u*Z@}xYf75k(pVYigULObUg7t)6FdO6kXHI5#Lt# z>(^OESmv3N=Z-LskUUo^bPb+g{ccxH3~ZBPdlNkx&9z=5CNMxxFdc@m=}k&^7WL5+ z{=z6&@qt!N6u-parKM^agQ*~jWNMf`eaYZC{-?^%R5oN|#2cE%6Y{aqG1K>!{n+h_ z4VWM#!n{?6Y%;5%#2^qp2|*Kn-0OW<^TGqKySv)r&aUk6(%T*Siwp3KpnOIrN@g5> z&sn^6Hm8dL`pB6O{~_FZc^SogvpF$IY|bUfRgT5_l| zcMXJjv~X}k`eMKSRp9eSo{m=uj4-!(`FPb7cPq8bRB=|PI{0E;<)_6MvDO-!kgHO~ z)gS^mi5>61WIFQycTSF?&uVgrx!D%?vCbK$Y*)Wie%uO2z@JVn5T6-5&Nx?9alpeS zl=85l$uF~^)?K}%jJKas(gYQxXzwq~{5F4*{zvCD(?KhTVS^1NnT}|G5xqArU*EqR z-gbC;uZINuNemKs6LJV{4jUd69Szf0y%MrRVn@1;yPNIIJqp_^pJPtS zHi`+?uNs@Qi}~^dh#&Z_{i5ef314j_dxF=tiWxWx8G5W4W77xm+W^|Wq2h~LPsrs* zpqhM(HZ-RjmVBCt$55{cYm4;R?m30>_Xtc=UGblUBVX97@C8^Gioa=t(HHXrVfwTL z(lt)ck=N#?)7wOchwEaRWcnI#C?Ag^|4XV?JAY!6dK$Rk7GLV-ctNYfkJ+TtoxNOj z?7?Lu+%J7ZHmT&VMN$MzP0{?)Xp4=zH`CW1Bw^)JI{|pfhF|a>lWv)n&Le`P(<*!R zZhH{639xe<{*yN>la(HNBQBa{sKNhv!^(Hxs}0-K8pYaU@WO6(hrA7|dP#kEwo6sP zi^q`LuU*F!?-i>SNSkq>H?_T47V8qy-hkORww^pP@*rB`rhes%RKqt94yOb!v@oEHsWZrvKhhG@6A?CM3EVR4h_s>2 z9OcLoJFTlbhqC(;-5xdQntBF5c0Vz@@Cr^lUX&^p7$FQ7yPc`w90lJV2&KvYo4;T zO)^;;PuxvOj|xgLOn5%1f0iH1_JcFB#g1PnrmDJ|_ouNZC&r)bL;SYcJ$yYU=U#1W zG{*?RRs~k>zVwu$4;#a~iM+Y_@<@f$}&#xSLBhEjGu9p z_p7Y)gd<4_2penX2gJplmz529DXA|ox6?hNRBb(E3m`|fpAW^fw{cG+ZKNJL{&1Ax zi2t-=nJIT`sUaHG{%EQKu}a2IzK%TDM$kBy+-C(hxG^i^bJvV0epZ~0FcZWDW?-~e zhCf?o=c^&?cK)XW=85K^5LEb$Q6HI2qx$(fv%+=>%pWdv=Hf#oOEFiv<)kDB+l4a| zX1|mN=Uu}TLvPhqkZmQm*Mnbiew>bGcayt?y4sl_Wg>pmTo$03rSL0w4vvN(G^v!yPa{s2HDa$S9_$NpRXFMmwe|Rfs81tzz2G|(c|p^5;c4`%Y1NTczhmfuHp&;_v8?;>g?Rw_{z1JRgdYrY(#LDRV$6cG0^f9=>X>)f zx5aEwsE4rvv*Q+{IfIHpXz8F{jzC9`oi_eZ$94V8!|8iI_(-(4|2$vurtCL_-|J6` zc>urtmd-PSx@VNPAB)vn%e20!W@uYn-IUe*F3nRWc2M!=I3mAWM~1LqVO*IZi;pcy zKbf_$oOYs6vU&%=6@kkxr7>z6{RJ<$*A}l?jG@{u%n>855oE8W7F3XAD7ej;pW9<6 zN0*Y-KS;e3A1z2jLGr)E4zc(ISsNZDbCm%Q?_nKHbQ+1)t#kMfL5-G{HJQw5*5990 zTy9)#*tD>@b95ljlL38}vj@Mh536VMDunS$S#&8kWOrum!~+aimHwXF8n^l9)eTs;ABCu&x(07~S)EWC7@*|zkkVBI%O;Y}jjj1(X z?f2v`&O5~s3mcAk$V1fDNRAgAk-|>-(e6u{427TKuUlvz2jI`@TlY7N)1)^_vK`nS z4-oBN*3k_3JYtwOw>36C@uS1H`{)Mq2BG`bkM&^gb*ozAwY8}r!_X^=~s1AU;$?}KD$jTk^`(iHL3K3tOd7-Q)Ma_cfv$m2l!?*E|@ok z3{pd8On6LZ94)YoSIdr*S`Te)f_ZaLjroj)L$lo{-JDSnt3kQ1)x6VINkT6llQ->u z+?~z|yjSGIW}g;uB`&rXJ3-fZ;AO`2y<6KCC!v=%0wW|WFRyzc8hgnSF- zU=Q6|io@8-6PvJR-gkLLGo3=~;1wP0NDP_%%-nuHolCJ@4X4qUV(aeLhrYs$EHohQ zH=M96qX$V#so1N#{a-#1X80oya=$`)9!ASDz6)7jV=aIioi4yv=FniYRP*m5M8z3e z;-S<(^d`UTl4%KCm^c%HshpD(&Y>)?nZjW(Yt zCN@u*rn=)44~7a^<-E1dco>rmKd0ysgb2MkQCA~s5EfMJHCVMepOGIq6u5ow*5b1y z#h{=(g@KH-ATeoPNw^G8seD>-Cxb}(LaG7z->l$jkH-x)KD9|I_0(^Awd^N?<}P?# zJJlvUdayFHYht4jrd#OY7HAUWn(Q4T?lR!oj0i5{>t>l{<2ZjqymBG+#v5;Q+XZRf z;9?c6$!K=6#!#D)vMSbX1mW2hdv*oVIwe}<^LnnGp{{EQq<}a>cSw0Zq{rV|V#t>C z6l4!i5XslyIH}x6fBf=CO&bBp^%l9IO6TV@$u$(SLEgC8d{LKxz^Q$$>G7`7q*c6; zIG!t~3)8&Gmgz(VFzhS-<`+`yW+_8mk&8=B(ZxSs2d@Pys>qf`Yq}@&=v!d2zv_-B&uoPr*aU=l$b&>D;cbeu(Zqz zPxW7uoKoJ8TV;Q}Qx8*Ct(GHyH6+UR*#s2rotQsKbsDJ9xu04xRCBh8Gm^@C>DvY$ zmP_7E-1BOpn47(|P`PyrCGF?X8gIG%Sm+Pxz>UTaNRUJtI#P1q;?hIP z_A;6LplN1swQEg(I+?Jtg~}!U+>%`;K{8%o+WkVib2XG@f?+pS;$C`A7Fu6sMn$%M zfU~{`x@X;&;kl`6Z6>0MMF(6pA1jAjIYv*mQ@SNp zPnk;RmFDy8PUHMZ_Ytfd@dzg|N!KW6>rKx&_lo+KGasI+-_WL>Z+%FzbE)az(XaKV zmL2bVIsd1R+_)K8Uy?H&rX<&a74k!U0{NN;+6cDRbME`}D>K#dmrpOf9L3)o)g^t2 zZKxR=M(I8d)0Kq?hUzo%Qik-I}E>r`ZNlNQmCT^aB@R0E4jc2XCyEE(jdonJLz)-2 zu*IHVtUBR-3%E;Xvl``IHNxh|_YZ`EJZVBvZ+(q>d$-v>Ox;}I76AG z0)vMp`X--qPY0@gh;VbHu?PUhbtN=>N9*mZC8x6DBrP zdQZ>;OJ7x1Oy3o0-ZHG%{nIC0F8U#M4(2!(^oG#y3GKT_NG{WHcDm0NJU1iGo8{Kn zuBd&xSl_w8&E3Z_iO=34G_ftb8 z8(8+?9|KGN!3C2-YT9_$u=BYBSn4lMQ#^b5-$h1eE1nKuHJ z^>`Ak)F)av*tCvzIg!mTFjG}Z9WC4F)NdNj89Oo5Fe(5{U8DQM3)>4%lL~j{M$9!P z%}Fc!I^8I^bpDQMe*y&ocuPTzB`iW|Hna9T76r-uP@j92h6}y zERE$yc($L7CsbL?SH|Viwn-n1V|e3>Z0w+SgIo}WntwF~TavIKN&IY*f%=Ymt9tA7 zbV88}_8^w&SXZg3==Z1m{Seu&#KY*U!N(-Zmv4dHC2ZwXDtxN$OA^ECSV7ADrfHLW z`*+j8wVt*zZ?37vx>jl1k60vDHV?1Vz+`?o8!Du|@UI{f zaGW4e_;`0@l}U@N`~%Ti+BwZq*lLN50F-SBr~ed)>XJ{k)+K0rio;bw;^`C@U?5&I z>tc;+{_P_nQWXEp4+9Y8LQiT6Agf+%1jN6-B8)5VF6;132~f$n*UaMxfROg%_&RI} z8_OMA0!?D!eR+-+F1Q^A;ev5!wy%fZnV+kdWSwYH+Uqz-+ZVGTXE#?d8oQGPQ1V>* zB@XOng*I!0s->%}^v{LEm!+3KkGGWn$souTqP4JfPM4ncHi^apF4gIMts*tBS}xh= z1OQOY6kjkpF`>@qM)7LdQ>_ahR z4Kpc+yA2y#_AB)(H;iNV8cZj_PY?Z~>AeCjT=ZBprvuX$5{?|YQ{1ri-eD@D5+5q< zsA9tSLTC^j*+Dy*qjUo>UyCn%+RXw!^c4u(BeC*v+oiJV8sTDt(9MM4-qwg5LjFzH zuUp2Fi*F|8i_hoK-r}R7rDl;*FC{VTG8a#8j^0QaL@!&6h>>Fi2qx3*h`aRWif2#W zzGwshB!enQKGUwL2@6xDou0V#n>%YSby3&W>Wzz_D-Q9(a-@)#vE1R*lr0e8Hya?C zl7032=>Ta#!Fm!^O`^hOuhorttM&vug(6T!6BqZJoijstwFZRbV{ ziz$JA{&C?z{2T+T`k!d0lVepQPTuv<#c!YwP$q=uBe72LGCrHE1jC}x9FaHU&O9hP zq~zlIghAz$;nBVOQfZ0L?6FrRyF<=E8aEt*`PA=5M|iO=asC%2SeZ8HN{9S&v_aq2 z1Y8rrC2xU$Nq_;|7Q0wqe25NOL%O!`i#e!E8!m0-?Aa?0Dz1NR;(B`5Yv1@l2?QyR zw;0S(M3TaV5{4E%JK#Yps%WuWVRi-_do`(Y+3#+GrdpJzx zanNZs6R^n5SV$)N>(G@Lp2M%get;?+r3|(VSQFex-)hn8BOzvq!Gq z-Sy_h6Su(>YjFOP9^JebjQRM?mBI(Kq9I?ExLJ_p;3+X z*BF99S8K2Sx*P?x?S+rlIa<3WB8*1<#SLUsSKeef=HrFqQOK7WFOF)_6aJzl(`if9mWcosjU_&} zA~j@9@*z)r#pQ3bz1x1-jP79QD7SFrT0*M0H4TM(R+98AXU3Jwqv$xTe;j~@dhSV( zuC=v@Dqmqp!77WmVpnZh@b@D>Iu8Be9lGhN5S04&0{wlZ#zR|7bH^cuU>cdJKN8Hh zSIGGC#euWsF1^<_rdDvdW>qYr&k5#u&a?aloHX8GSwY}D89)D=0oHK&Piy^*h7qjs z|9|Xe#ki&0x31{*OJAU|WNLX-t<6^7G++XABz0N=Aq7X}h;J{!s4q`SUz+pvy+}6> zwqfLr4A0ikMFU|dnun*|J zqJORP6FJQspQ76yZB#hUt}Z!Cbpk@A;9lo}z{JR7FKC%~_U;RpY5BYIRT?D{z)ctP z(u}UpH0{G);&B+0SA0S9&T6wFTf3qNog0huqH3Ot{Ug{}ZwJpq++NvW+i=!*_B`L7 z1{5I@*HIdy%-R28NmhBb-HOHJn~CV7U-=ecUGvL-61O`Z`vP3cb9?a%&%2=$Wr|>b z9CvE_du#lJAB}%*P=3E{A@WHaEdyh$g&y~j zIN0yWKUjBbg20+8H=&AUsSKdL?)?KnI#dh4CcH%K--I9T#=C{5uV ztKZ+W?UzV{-vBo#rM7z(m9IN~S-$8kwR6$>kq(lA=&YzHzXGU6NJ+bIPaBu#sO4@aph0VZHnj+}9s81ed&HnUtR)4+!)uqi;wi6GGfS8TtqiIghKQ@J0B^b@C z2&;e)V5I!r9*L@}rfM;N)soxzNne{!$?1iTK6y$&QpDu*==gQ>pT-vlcsI|sh#nhs zZGlS}<3t1kzPU~8!04g}72ZJWhix?O_yIvxSp@bnFMMru%5wB?eVVa` z;s(gb<%I)L$Dof~7GbrUgX-Z`O;5?+@2Wa=wcOL>JkS}eFemD|_~dAUxs*9rFg-Zc z$=wv(N*V?Cv1c@l6VN{htwTOXO;uB8VRcU0vCu`A(|)qurpJoI!_)J~f0#L&8;7!j z>FVuagIeK@x|{1VdL(@e1Aq=2mwat9PX|Dk#GC0-+6kJW?==}^1B|PIc~+k1u1*c# zw6K(%@GmD~h8=Rl2NRdiJF$ryYoAe|S)G3$;u!!s+@LX1zYH8&;rYtLR~Sdy8&uU^ z5-)e!IVraL;qdX`oixq)KE3M628Ty)in>qKW^GbK9s_M@$iMGaX|!C&9A3G)VbfTt zZ|Z7Zb^M$oSh(Ee^olu8RhL#OuW0FEl-VWe8wTkc-B(u1-~B{(XK$TiE8tNtz6=a< zw9;eLE-8UCz`(mpI|h$TRJUFDd0caR&O$ZSRaE*|Yy3I3x~uMfRS8Qg;T%r(#OSNX z1{M~#8QVZ!i&S*xTG=+pLOTs6F#d?j(Rr~lzd3aSL@$MwfcgUHsr5M^o~EY5a78y9 z-3RMCiGFh{+s`->g}Zo9`!jkVi5|>-TrKDok2LKyU|578{kBL_0n#p`9r5WZqB=3U z<>Nlf?vV;i-U8S)+|pyAI<(^f8%E%4OtwOlxexO3NVd-c#ZJqYHNRY{jS^8S?6f$O z>CA-6d}-7j?i1V;tXC65fAS|c(0y>b#4u8bPxqIeDQhNT)MiR(kKr2(YP~B(s9`}X z_bz~g+B~c_xmRIn0X7z?-AFL;T?ApbC%g7^b_&=8b)%VCPDAo;pPDa_(f=Nzb%YPZ-u2TI)sHxYjm?%6**aeLnu@Hc&}zFzt{&GE_cEhlHkV+jhC+U)Rn z5>P8AtVPJ;JK5v=un(Isvl}F*+8FUk8uQ^!j9_23Zmhxg5C$)WeKItwee|Dt47efx zvfAG2&;30u`8h}MZYsx2{s)V{;wk0j6oi^RErjx7Bh+gM)Si}fx--q~ z`g{1tK~?^`keoY4htUQ(+@mIpY3~B3%H(qG21k2T7E!jTXC!8`{}5fBrr!GeXXN%m z$dfRvvp6p+(XHm9vmYH^b-$g-+Op*dn$LuEN>~~GT#7-RU-Qhxv~=WUl8##A)y{j>LsIa%`{N+Zc#h$Zdmi}UCX0_` zcYd4sQD=WvkWcG}I?uVRqC?Ue?W&?*HqDNuf4l@v^S$c(V`sHR4(U5&j|}{{Z-wrz zrxcC?xV6fn&ud8IP3brE3}&v*+fTfY2FlW3NnEli4jbE@`L54}wVhh*Y3S!ahWBG2 za9@>ioNLuE)q^t~R0c!m7a&Vn7Y8?k(}Gi}q8= z$kdBS8LwO_#canr`R2C21DGL>et#R**Nh(8-A$uY?!y7{qqr3Ux$p?0xrcbz9pvDg zv6XAM?mD>@r%uoKlB}y5;|7lpxPKhOakRf#^q3^*AOIW#dkQlD2XQt!XG)tXAS>e3 z^FA^v&2!LcloQZ5JNiPXX)H4{uK@;`gmPe3&!0Azfu{mxj!o4V4DtxtIn={5G<4zt zI6Me%xFV?|u$MsjfFLnnM)+ZFmDrBrUj$apbma~h zvR-*A(=JPS`=T}2LrZa*St}qS(}RqBHN?B3PcNOo_g%mXm<=i{iiRW@7WkTdIk$NL zi&$g1B955zP&vCf5%^p9UrZ@9Tyu-dhtr-1KU4;Zo3bYmj3qk&37* zNn>+oF~Iw%0-_(Hc;V^uwIRFKRn;G@=$@&(hb+&}yQY1`|32K>ZRin2vOcwl2zcW$ zzI~2*?6N-|@8|rSb`;Ah@+R+UOl%%RTV%Q}(}T%X+$H;0df%6JrVV5eX~r!c}o)ir2J&l!d=7qD=M_Fs+^;#=J^b&(Tot?8}j*Dt4;IHQ7`Y>+Y;*Lw4^PZ z+!ydqzvp)INg$c%ncV)uE8%cAQly33>26!$rw`>i26-eP3`G9ORBUtQ$H}N18?1wVw#(8>D_hyAtw%iiIb)r(t%0>=eRj`~|OjRPkzY2#b_r$hyBfl{8!2(B1<&CG!QVVsd2X^bWXc}~4naG8C zw-4J&q!-8D9z=%t1%x73Ha_Gcm)PwVHU(^29N5gPGF2AJB(n}{0ul@=Piat8%Yt;_ z1(X3sI5WXygfXP(gjjY-)M%QLVpMI5CjtNKCP%{t_#01@5gzMsQBHK$BHi(0;2?G9 zq+R6g272fstllZO`zts>Jckl6pttVV{h!EROoy8KhH7ZiteeN+6#3U7O%0dPKKu3C z-Y%w~k9LBucOy4pw@(2l^w|Sq|Et63v|--@8=^@@GKil^f@d42qQ!iO^b^_5>f9d; zmjH8w%iw_jGab3DX0#a&q{RyKr_1xZn6+$+_imj&%`P%sxowhEX)pHjI4~VwN3e#_@Du; z;_Q{rhkXFX*HWTdDNRX0(LA-iw^-?xXb4Z<)#)QwUTWBAF5*9I)d4H%quf4{_3r!j z8h?rb5%$-?QjZ5tYq9x!?8om&gW$cA;KQ=peUs1z+RWQ3Ve80gQ*I37s5w0|Ch_Omemj_5x4H$0R**^0$~x$Eh!4aVsC&r!IhlK`$;^s@_zsP-ZdRENRF6c@(qN zrp&had+0DZ&5=aBOruWBXBi}l*G+08-ImWI1E-XDjF&Z;rEXZX85J?>ou)Bhz>#}- zxskO@Nl(CM1xHtL54y&s+lKI!lPA*z?xr#R!}*V9`O~F2)YV*a=nsIb9rt+8{`#nU zz{J_s`QSWBy-Im5a?K9u9*_VQ!xO0U!pWM>LGhvsfF~23=*W^5 z*(?<$H8PcJuBgsdB^^8_d@`~y{W>j}UY#vTQDWy6YeI$F!wJ9!#>Dvktmf>4GotY|tyZlEhJL!3q9AflzqZeLOwn zW;_c6t8!qn<)#Halp1lrm3)lVA(w@W3s%LQ6Ti!?a=pT8`VEGh=-ov+03-b z0mqOz%~LF6{yMP6vl6{X*7!f9ePvvf+ZQe%2&j~x96%67P+CExOC^*J=@jYijN zj8$+R7I6mXJ#cqS-?jQWmhUWPZ?Dg>T7bd?m+r)&prs5^q85Qw=cT}WXBGxix<)3B zI)P5Z$G%T18{ZiIU0QH0QDDtFU)-N;E?~-6g?Bcr3<601noZtzPb&pf&uDModd1S! zaX+k}`2;xPYE)o_mUr1d*$Qpg$#{1htNThbbc?`!1G;x{5VLrnIvt-S0~CX}ad3cu z#+;wGXsgb)`D_wh=z11i81rDq?NIB)@?-rne#)H{nR!d=p z$?mndZ!6;pgk!2Oo2i zO$|G!s}}+rpc4@OI)C>bY2^S^P{3IH`}+PYwqMkf*i> zM^t0leb4rP4T)LyAXV1>onwyNzu>qF3Zox^G_}AS7e&zn&`w6Fg=4zthp)={k4?=B zcv&EvY02F~sZlC1y?BwkN*wBMfu+a=d{ihvrMR%rKzwo0PuHCdIJMQ7e(fiVgfMQ1 zf}PaNndiO(%XQqk_e3cn)}TAXPEZFMuJycgKM33y(Er2etxL42s)*86E>qQVxgk3T zUn3BEV7K~pGkapWa)r1qH$UGObYvjtx;s)fx2T$5xb;EbqH)gFy?r}+Ys%hreS%hT zZ&T#6^$zid#blmJY+OCa8CQzaQU@Az1`Fm7Pz$`fjN?)U+9|9I<=EsJHQNi7ZO-B^ z?ZFdbTbk+RrRus1WK+6|RiF%P%*ap?c0nlo)^~71%&U%BhfVzbWk}dt8>w4`hg&Sb zd1a_Xr#k@i0gZ-gIvc!MVb=P7XwQ4+GlH&dXt#BH(PlX-d&w!^T z4?Un(0OcD5vjl{4kAy^~b4JczCDK%~g)Hq{g)el@SEZ+~L~L{ah`&yNJ#at;5@{Ii z#iw)hitkCk*h3o?sxkB+{BSS7(Dh{qwn}%E&XIN5MYS>Qy%nO$G?wT*T#IE~EL_Y_ z(f+jXg;iO(qPk!^;ipZ}P-C?^V31nDfZkr7*gl2JPXGbcG#_49<0~X;Cf-c0Z11ysmrn#k_qt zEQN6VBS^YEcv{L!w~+LQGDMV6+258I-MaPx1Vg*sBTyWCl)=HnB@4MCQLKojHP+I+ zG>S1ssCt@|g*`2KWUhpPx*j5Q^Rx0wh%P*bL4X2Knb0RzsxPAqJfP!mh@DZ z|4$&xdRS8$oG+6vj~!Q}%(#fpG4|$o zuNY!NP!Sh8&?=V!PKJx~Ze4NDd2Uej$77sIyEjG9n#rhmi~02`6Z2y$(m+bAuwj~N zQrqxs;)^kP(+;lLpP|%06owW>j%u|{8fc5hsaGkOp^=!Y0%Nt=ljJ<%i6J3Fr;?t( z0m_hqYSfMG+?CCZXe0YI`rh>}LyPs6%r@g?0wMG+kp$lB5;|V1v~h4Ho5pZD0SQ_n zp)VbrfoHpzUoSJoo-1y~bRC?E!YzKZ)6@U$rQGb|=L z-DaCrJ9Dnw1$$ej;H=lpUU1S%F`I*QeonmHWl+xbSkm+`VKD$gj&o%^Vs;)FY53xz z$oe;Mr(k8`jzgMy3E|E|X!>#dGI+vJXC;fB_QK`;DnpSb62@XGkkMIPxMb{n?Z>`$ zR{4GZK8LGXM2`(rL_lmUcp@<0CIa+e=_V&bI%Sskd5W9-6&MOvr{JNDmkc_X+P}zU zJW*dHqD+${z#~Q26EusG9i_Kig-x^etfVeH3##=Sk0gwQpKC? zL0w?#Uq!DWhIS8HhYS1tLGyq>9lnEj8}a83_~&Z3;0*VR3W;xq^Or#sM#TL;n)-m}9%Qdapk|_nZQkA} z0B}5I!afA%pB38AMzazKL9LA9un@zC=#>gS;^oQBJdnZv|6XY`BnpYCXHL(b>R0>A zRq<4&l~YYf%?iOr)N``oHG<^FOPa}(jq5Hl0bOaLwM+qV->Wy1~@RuUE-ekFv}G;(A3 zlUDU&^s{Hrgtu1b;>|~^+)mZB+>q4hOnEXftkc`uYyY@5inn|tKV&ykJOUg}MoT=2 z36SjP03Phr0NYGCipOPozm26y4<7R0gO7t5=A3U!Cx{X?oC5pLTUolzWD6eO9oOa? zEp?n>1**k1ZAI2_wxGJ?Q=i3s1Ej#(iMxH#=p&imU4sr`4y!40j+s!b6OzD3Zi7rJ z#%(*VqFCx+4NinCn$4DlpNi2`@BqMVrKzf4Y;P$n^Ux70ApnO5G_# zpR|M`A-XrG>X*z1L(z;396UWm;W9AA#jT1!mYy=_V%CB#blNiI`XKtoZv;DxB*zB{ z3S%^2%D6;*Nmv@l+bQHL4vR zoAu`N(9={EgdI4{^TIdEQMT3=*1Yqu?^#EP<45wkx(u8cHZ12do6YP8+sy?H5&G$* znTwQ<<8m)~ox+|Om|5=9R?8jwn%dgmXS+9%s%-0PYHqjpu$T<=+%b-cIWcTLE;PAx zCGFk8fQX2Q$(+sSnYx<`9M&`6vlW_7QXQ}mZJ>6=Uoff+jZrB_c%nIqag)5tmR7YW zXAN02gh{)N@NLlX7$Tn%=4@|v_15!+p;E`C998w%&MYPMW$lyYV?Ymz0nsy=5kt?V z?D+OU(2Mx3I9}7Y|In>zo*Wl?SQ4t$-IaktOYN1JPJ8dk_@N|Wc8rsltl$Ta!k(dH zl?$AnA2_YOzv5~1#+e25{OLV;_|1wr==T>i(Q_82T4RTj7w#GLyicqOqjEQsxMogCd`J^>5e2)W-O@y31zB}pW@wX2{oHBCXQP|nphenjHF0Y#q5fSOs zl~j3=2sxRYvMU#Q?;a>_1m7V~MsZx7{at2PY`ZYTW!*|>!jjZQ zX!fVx@R$?{L#oB99OX*Kt|X2#F^j6=fP@^3Ii+V1vFiu{O{uCr zj{$BJdb(>QA*7^kTU_Ks`ykgWZd`Y~{|L+sRiWwD70T69kpw<+h+xRSqCoS8G}}{C zNVD}xCQDN;-2FlMIKb=~N%Gy}%N$o$0Xu`<@Wwzj)uGy()Se~{E|YKA-L$ci=Fzkdz|>(R<*TxovFeyR)-2_*x{bUv|7Fx9e?3@CG!s6VFLvXtEjb>}W8a-G-` zP#OsKu(;5+Jey5yKsultUP7X}hTV#BsUeOG5#ZWZs)-p{`I2y(K4h zy3FLD(6SqIY;4T#g`sowSh4oAXCFq2fJx0TM4djZ_m!J_?8{*?F)^)kKEGqPLhYNZ?!jUtdejg|%W)IHa8=J-h6RR7=& z7lF6R8ILStS-8h1qtmiajtgjt^_b3;ALKAeoL~UBX9*aSKm1;B1g0E&o`%1hcS6^6 zxjkkVmD#&fpb-`UNXwaBT{gViz80?@)&BI@I>%hn2>GZt&IZ*+9w$4n8psb~9mQ}z zcZJ&lcsIkZu}BtkVx;3A59xmu3Apv6MNZoI_;=pOjR36aJj`XF&(W&(=1Oc`N$$@z z)ZHDFjOK&sdHi5I{%OQ-WdTlW$sQL`u{zgZ`X!g={o=2WyyzSW^aA3(t6Xss}Eh{V%~|Z@vul=iqlkmlmI*oH@k!8Q4qugIWNvYfsVc zm$sxZex7+_pr$8g#A{}A><7V5!)F+)#&j`aBGXoh`x2f|f0Db3E|3;sN4u5E0{Cr2{R4i;j({r6 znI{nJ!%utU{{g>&n_18f4KG6Gw4F~ist6F&iG>Xmv~vYGZkJC>>R zkJIgE1Z5U@N(iqj4^CNlr*X&SL!^+4q24dmOJV!~!S1e@|M}-S$3z)Y#cZIss}FDR zA*IYC@{W$@xrW{MaK2c5w+1RtCV8d z;HC9dMu@auq-$a`pWNWN4`hOa={14l6TUEQ|;~rwI7ml`f{^=OOrJ&LL2TG7X_l zR#yE=XS9;~jlbv|608FfyTP)LI@lL6R=mEEIfR)F z3l@~gc1^l;-0(pp*%#;mp_s}tjF-3?9QPRzuJ8WRUA$TGnd@!#c*ALhdjkr1C@u!T zm0+<#J`Rag16i66B5mYSWNSIa?A~6aRDtsyZx;L}xMFu_tJGC%q307tv%a@C|25o7hhsj8`xsf1Dx&~8%gmgM*t|a6cEcuRPvF$+Kox4oF@t>cJrBg-79oh zo35Y>M@Gt`&**xlv>A?YHkvWM(FaUzl0LkKDbD8nIb(agil=D`}g&X=;(^{ z33GGv36FyHb-S>WrX06~PhUToJu5pctsxzWv?ZB87yH-Pq1ignhOpP*(8*xdrhiwD zaK*4lVL$-EXuOJ`;A<}>fhWi04?~?{?d(kB-R8I|}w&`D=zL_?pd!|LiDTVj5XiMgKt`o@hLAL?IL zS5@`pf9$w&s8h&)qSe*cH!hS>Qc{|bJ$m%0Em7iXZ9_xA)YR0QHIxH;d^9mob3fSP z_+hp8adbgJkCVN;l++KoH4%ow?{@aVK|wmHHd2bmh9pbQ$k@DImRep+cIVEW3G&A8 z-yc>SJnd){_kjVRh>O)vD1|814G`8SsmYelmr_1@6eG&+S)CcT1`?A37%YT~|FkSW zj0W>Ywc!xUbLS%$p_~!njxZ;^bV=JsP)g+RDX=#elTuQcbYyk4wK0{4AIf&OD;&PL zKO7V&J^tYxK3)R;Kc~byucu>^wON2!G93{}rS|z_|Z{0FWwNp_E-}uVN!!x9Y6`OZ7 zE3C)SIXN9y%oOG1Cc5sC>OoK|`5S7`^*r`xakG5g%7b00si~{#%K^n8$$fW>G2SIW zju16plH?jP2&g4jLQs=Cf?7_X?9rVC(LCn^f`h|`ABjAEJdhF^5OBkJbJ@3{ zLG-P+cSf_=N0Xzu@4u210s>%-+H8!M^#^T_6d<(CQUf0FutH?9l`D#qZ~9eDOCAC zl@31(!@xg=x3-50lkF;pZ|*-h6&yz|`qB${3XnVlD87T10mh&CRu-lLm z^dIBp^XJYD%;ORg>Xs>Z`uO0;VBI?6DiuvA@$vB!d^MjxGvy~tAqiupSo7UL|Ld{j zxX{H+w+vKXxOt&&mp&7H^yrQnlg_heJxNcFdB)@eC>&u*=)5zt2Hh{d`26TfbUK>H zpI6tRasvxcZXj5IRL>^3eui)9K*hn3(0bIU_-=_GGZg#kNCaS>UNa6Cb@rO1qM|;- zw1TYcnB4Tsm#CH@zj}HU9PhqUIwDyL*THEW6Bl_pYHD*@W@_q3o6FM_At50bNlA@s zxv5W8)k8l4Dt?Lh$WhSVd3ZDi`(C!)R{m%Ma$ukHhf*pu(clfXN%9JI6|Jv0eltZ1=LIfHx^ zQ0+$^@Ik!XR#0BOdNslHV-ux)MOZ?D6;zUR$ihv<-yVrZF4g;}C{w3}`o>1w_BicokKwDHUzC*E(V4sCm7Myt3Kq1xB)16EF#i~ z{7ze2+k~YX_LhTB3Z2H{Xe|$4o~k|3lX@Y zMBs}5t?!mfi^XYR&Ab2zb)G_-apU8@q)}Mo(02zq@AyZCF(loL)lpvlJQ)>&c0P$2 z2nC>tnLGIxO6BqE=+ZS$-X2ivPPtQGR zJkMtEFP+dB(2oNZh^x>&IRtcF2L;Tqlp1L=q>Bw_^2mPX~DWjDOp6Tj}$jJp|y%JGfL0LNeo8Ok=m02{{3{r&yFnc2QOT+pLY z8UUW5YWd(XA0Q|If}Vu&XlQ6Aml3?IZGC4*CIYR1Y*Jvo3-^e*O4YiY6MhGhTa6agYJ@s;$snZ>ciN;ZL}WmSMH}W8vt9xF#wSdW}o!EB!B76DLrNU0CYuCQ0P;uS6m*;I*KX<78Ht7*sE1w2vtyCjvX}{&^_Ud()_$|3( zI_OYtYydsZlQET&M>V37R8cW1Q%O%sGD~$%TVzv0S_|Bv8-=X&S3EZk&tp`l9EEaR&OKh8II4q+1gEr3ZhNpqCpJi7)6Shq=K!KvH~L<4XI zfrQ`wDGdUrc7!15aYwD@G66wB1Oa3c-+TG^xFaq=f|P+?3&I?_l|kk(Z-$QtE7e-# z>Oah3OA9baBuYMQWT-|rMi$169zTBf+u|Mt#W%t9n3ThDjv&<5?`g;!@;kveEvODP zyrkDDr!~zsY=VC`SGl$q5qwR>d#C5qXnes-kD`vfSO~ zbt3`;wHt2J(9l?y&`aW|&h6McS{ z3azD294VEc5hk$!GKskoA|n2Py-oLKCpIbxTJJxcybWz}anW;u zeX|&jhmWs?-6R5FW<}F?BISTs6FjKColzdQZVes5#Sq;31nFZBmK_%pB``41FCxPD z!dI<&f8i<#*Di#cK$<%!_S=612;9xJil9Q8jl^9J9gW-1gHOYYj4q?2qjR0N z4GwxpU4z~-Mc6UV1A1&JcD{|;`)(`V|SyWt%Y@~Xl*F+zF9_*n8avmcl1wNUD*Imp z0wOm6jEOxB$nmYBR903_)c<_b+1~b5<9OtKd6MKsMF8#U0nkDL z(9*76N`Sgx9LUpw>~KR2ERpm>E@RT#RSY;Y~?h z$d*_BegqQAE*RVniTPPj{US{MKAwGr^7HOHZ|^**`a|a35znWs9S=+<$&1mRcX14k z&mE7aI1!`bdjKM7CVJeD9l<gaD~JLsG3pAD0^!7a2A2-ftuV85kiA?^K-kODS2B^G z|H2J9lnkawuhwS-UT}p)B;LTrT`*Medl1c+yYWw&k4gtwH4ML};HSw@ilVFm54oe2 z@;?a~j2Piy{y|$HfZQ{>z4X-2Q)Qk1+|Mo`^va{c)ZnT&=09A_3gTjO?QLy0n3-i9 zR%i9IgQQdsIHMdQlA#v8AIZ?j1EgQWW8yED23kX2AX;44fw5^t*?g*`tHP*O@`11e zZ|6n};ye&cm;mD$QW$_5%;=v(`vI!Zv3+RcuY7BcbGi@-bD%t^rV(G*jqS2Cgva-V|9Yvf*xPZ})29 zU}4FP%zrtvZ&Tb@VcXkIqYIl5N0YHKHK-=f9UFUw``3N~Sv3H~c}17s7oq`q5Fx*L zv)ayARZ?<9?sq`}+me8dlT%44vEY7D!y63*;53u~;KCjP>j=ne8l$U=b5LnVRynk6zBou4ljH|k@4-f-6x)!~q2;xLU%-QeWy&Is3c>i95 z0iTS7#PBGjM4SbA0KiRp^ysa!h`*no=XkkwF2P`a=s)NuU-UeX0(es-C%Dsa-;@sj zZ&RwrhGK7TUp3r)px~H*f-_?CN}&8(JGprQ~b$JXRhDT}_7drFo+dY{BTF1~4v=EMN`l+~1t954xIXD#I zDJ3Q(DA~!s13Qe6u1kUOFkYR%<3PitE>A{HTSbGFo#=`N=gRU|IyOYTBd3qpl;;9f1+*&MEx>`cOJ~` z>p8CO5VaF}F4_fiiu_K*lXQfdd)t{=7IW|KjR0W-)GpX9b8LuezC^0`KMSi&{_`;? zigx9nQ8b`{fK?5%{;w1GSB@A{E?q?r3!0E<{}+QxI&;L|@`-OyQWDj>!fB|AOj-Ul zHLz(QyVj`=Ez7=l@GSGtJ9w^o=pDeg5!9_*=pcZ$9r)n`26O}I(Z>FHWC_nXw9~VoeRq;*0W(_;Go$>+Jo!q2@yoDPz5fdE z25HpgS0xPiYCy?#jX_GT(h2+*_r66OVaG5CZqd`Tpj!a8S%?`I7xxBY5~}}H?^qH* zb^chPrr%d_ermb_CskYps@1Qcrj)K;px9sJqcD%r?VF)vgsfs7K+R@>AIL2V)_pf^ zNs@=GZ5W8L0Gmoi@o$@Y9aOt=pm2mx0o%fjYQ<})dR;{~6lkHsCmaW!Xcw7R%>KTy zVIC2<-hHIaT>l?PzXsccjvVO_mxTQXJ8FX#c%=(*JV+=Wjxk z1y$abZhG`xcjJ4BiTXLBO;+Nd7QiZ{DX&YtwY3#5;ejQ>w9RCn##`sB|BbinGGGar zwjzMl0cw0%puB9kTm6K;aL%h7}773r!l^A>Eb^bQ{UjrBf}i{>T*{ zF4$6j4Ra`xE7^^*`I8qo$>%^=E#)F`PtPL#{{7op^~pTKmBNJ~yi=r9;GMF7S+zgaTuHBdo=zqUSH-iO=6)YA~VkC&8| zc2|D!S1#KB1zNZmQ0oXu>L$qe0c8zRov+XT8(K+Xf4G`I#MSy|hC00YC2MPIa~0|5 zPnTjstfz$!ictvvxy`yF%w=_KHEAivw-lGq85 z^$~>*PW+KnA{%@Tau39>|67&eW`SyxKBmX#ARd8WYw(LdI4m>%hl7yhdyK0of{9fP zV-=wm%q#yREewp7+Y^8Txku8FCI90fFo!1{m`yzKsf9nP9ZMeJ7i{sl+HyaA`m}*x z^zGZXScI^kzrr+N7dk=!#foUP)Pl5^3kI%ihmXsju8qd$Z<{6ORTf?azxI~^-306) z@;pLywi0!DYkzg7M@t1+^w?ND1WC&J1gWpWudg%HuUT!FMri(fMtp3*YJW=>>4j$3 zh{!eJJ~(g%;x%PtWQ-4LIHD11t9b)pixpT&CD$r=CU6A)&fJ2P$4$Bf)Pj=c1Qqn$ zQq;ROEkxair0k3r$f7&4cmhQXGN3dFXbD5mk`dmAmjAz@l_Un9+Sf6m20ay|nO>1_ z)N=s8jERZKXumS#VI-8Zhum!hC%Yyvw8ZD}4{&Wlxw>J8mz&Viu86vgYrz%WB}u`8 zlJg{gWfi}mStjdH9NAJjK*S``g(BG^|k_E`VPB&?ToNg+!%AEt$98O^XIWTq?TPljIEq&|7 zlXtDZBeNX<4J3;Y?hWD(-wlXkW#xxxsy^UjfJR4mc|Gr!uHx?|nvJCnPI`urKb;l7 zUTa|~_qmhGv9fL!AijU>xd&LdzQ`TH^rOZ669#B3c~?>N+D*k9{K!jwK3xUgf^E;+ z0iwPL)XJSNu{T?}V7PDPan}LQ2~9i-g_HXM9c0+kp8;DLA2i9(e2aJvxyGvsYFolN zsfN)rjM*^?e|ohgedJr%V4Rcs+V`WnE9)U*3P5OnV9v7a1alKxYjkH|W%}kERdj?{5cyB z6+c#7hrU2g4K-QKf~vBfO8#ltsP`T+FrSt6pQyc?C!voC{ulk$bf z=7=A{k_NPs--g!f!0$rEg)&FRzhl9lJGUw!AYBHqQAT}q2YkV;5{&cm3+XG+2>D)s zr89Xy_&E0Hmw_@_x&Q*F)q&CgC$LbkCf%~{;iZ@*QETB1L|we((VCWILCkF#=cZJ| z?*jsToj$9{c^a$sKQ4yo`Nj~=M{SJ7qVyI4{*Hu_fl zc=XarR)(qt*||tqevbXbz+IneXB_LuA@K?#3pNYjD%J?K;^G?Wq6b@My(VRkI)EC< zo){5ckJ`bd7(X_iw}71L0aUmPHW z?E$#h69bf%L214e&PbT251_}0^ahGjUW`#P@Poe zL&L=daaM3^TG9ge;!3Hzff_i18%iRLf?y@!vpZscZ5ltOk&67aZ1+|FK#?UcxWYxQ zTRh_4xG-UGMW8ipJrAp=d*pt#Q$9qGw|x32aHmm%uXd><+{h2S?6t;pOX1=`Bm^ez zEiWYsH@-m#4@MII_$gck^vcRwJQ_ZY9d&T*memsn8D{-$N=Ql56ae3G?P{(N$OQn- zQH2k<(d|uNIk2u!Y)9G1c&);rKkxB|L1pk_t7H8$fgk5K^467mOZ2lpTB&r8jL)p<3c&48ezR)9os#a@N`uwfEt%dT% z(Uhx{dn-IE;6Hb{TXSEjfeD!S@r#BUn7b=U!yz_tA^m!>e0$LyililANO<=!y>wg`ziQguaO_ONF$uJd$uYfLA1^ie^9{{EF z1{ch>wyJGW8Y8jTtkgg>v}o+-P+0V zS|N+EHFZ}U;c?GwiQws_1d-IPbQF##|5?ytGHbQFO?^C)daWkRWBA;ExrP?CWGHR6dIWH_a=4hyu@Q?b83^#AE0lRnb?xq}J}*JBoud+m~y@@d^|%@})y6 zNrGqngB;UxkKDyli4`We!%N3xYtW);%l9H4U$iFliGgyP3u*FuSxO1s5mfa<%9jJq zX0yNe`MI%ng*$C$dMj$oTe+$(n6~UfU5?gMp^%+lM(oSK9&D@04`YRs0O`!v@0Pk8B z_Ibky!n(<9nbyC802iPA(I{svTk+SqVB7V4WCr%`P)Jr>y zRHVanxJ>n4T0i;lCD)q*^*n);nWYp`Vxa#H{M#8hcWYqb9&9^U8Fp7}_I=bEoB2ZV zP3}PuDh_bfSjmB8qQK3#o5J#q0>9Wc1kCEH`@X>Gkyzf72T$Bu$j_v!3#EUgdhweD z_+VC6m;FbO&uvFT?E?Q>iZdrfoq;q*?v7;_Ozc?`^_{kiTAt6A!8JpUZ>PB6uC{N- zT*@Dr>^)1*WgSo@25tHks0l#sI}3cNGQu#iqR2_C(6ik5IYEh{2F)^QoBZZ*n?z{7 z-n@*?q<2Ks?iX|S%7stmzl(C=&&DdG7F&z25_LC!U{!Tc8Tw|ZnCiN%_`Xc(YNB8j-GO@hysR!M-P-ciHp9hTzxc&Q`QiVxqpbQK zS#0D66O}uH3zrH-%e{VCJTK=tpQ|F#olc`jnZHU(6VDq+d^Ve`NEw@R@pgHB3o@bM zt`Y^kQl6P(p5Zq>bjP&qWy=kt-o(Wf!pf?p`5sf{sCaKo((#yj20wg>c^WAxDP&i* z?Rx{?Fn4+Uy+uIv>{zx>FiCSKVU#n&rlK|1ma4zR`|}QJx2hw}L@TzQyGzTbW`ge8 zN*$5oy?3&PK9xR}3wEMZzvOajIQ^ENx{C^B`Ob1_EAku;mjd>5hoU$aoNL-;m$L1% zn5ZOaENru_l2P~vXJ%~*V^DopV^|!x^r~%V`!4%Rj7Yq-5vM)x0(%H^TDjD`eW}U& zGOff%vi?+R3)%2tSFg=WmrAHXTT9upJ&RXHO7c4>BNgAg19nDn{2$ z7wF3~WOSS>wUu0%7<)~r5nQpk;7aqX7F}Smu3toZynFcVYZ`~|Ir@~rIne*?QhICKE7C1(6my57`r>tb}_qmOdb4IX~#_@6pWOIH}t$RhGXK zM%vsb!&pw8Nxxy{qbn9Q+#%y1Cl%|AlALe{>3h5GkI(02Zj3ITztFD@_9!u7P+^Dk zXM;bb?a&yv%e>K`9QA{RP-7Q{Gew{3ADzi47M%~dtrB{#*Ro5+_`jFZ!=WnQF2F;l z%;{0OFjX9%Sr)Fp*YW!gpE5mWHAt|y{PFBX-*ir!+RYT#@nA8|xL=u%?U#bn7b|(6 z@DC5XtJo}XjvWi72(5oR7c=fQu_e4zwb%LNL4RZaUf>gWe!c&&aS)+v-|Q#vj4d8? zPKw?Qhm~68Qh~FslJlKfdpcV+S))qz`sfcuwy3s*JLboWZD%hkDi_KG?I=?FxMWMo zrC~M!RbWiHQ*&8Ra!1=fFvqmW$4XCbXKk?i(i4kHW!L(IIi+tEKFrv9*I6m^0!FoB z;{-Z}1d-HoErYjnG~K1D6J2|*v|TWq+u{+*e^1eOHwbj1V5;k!oC(YA*Ip%rkM9iU zKjK$HXHT(SDQDRo_$*_xk#(PEvwexqT)nximNaJkwKu(hYqzdqA!+j`o^LYpX~Em- z`I`@07s1t@DjazxMjU!X$~LopZC;#Xl~-BopZj^izP%F$px888neVwTjcq_(#mDhumCENU@o7pLtm zt>O|{Ah<1Fx68Gs=cAv!OhUIAGj2-Jxipb78sB7FQGBec+@!lfjB5l6tonVjpRYCq zrP(VAR?JTSBHYz~GoN{ZUjEyH^1^>K$%Jus&ip^AvxfF6c6Nd?&)}ZDP^#|(L-Wq- z5L6(kvBaNZuOHe|*i;>CX4~24hxSJD(c{zhKkl}?UyH0=tGRjA>E{ZOMFyNwUX3o_ z>nin~PsZH$zF4~n-JY`whcAS5g1ep~uV|}w!KH~fY(8{vzLqXGQ+uF6Dz3T}V4MhO zYf$6j*B6$a73&Mbs{`3B{_7tjIO@gxCu&+%{kCo1yso=izVdFYA;jqQa0X$nUb|cd znN`?b{VtV>L35Aa)u4&*%B8&;eCFzUze&sOt}p9bnKHVCbGy}^9B5! zLdN+irW+M>Lt=op^@B0#)bHBG%&`e` z+xo<)V7rosXI33=Vh6j@bk$lvm`~d+oW2%kS*GxyeA+U5yzG@k7=Mc4k2B1f0m_@J zeqq%^s<9PH=rVdE&hUurM*a&KYm1k6*_+y%U+`#st7`teVc+6ry_f`RW89H0N7gIg z(HhQV@Yft&@Rc9lAdhBP?UrZRa5^iEjAg3J8$iK4$veNI*kZBF9(GSBYnf|LG;_~p znV@S13B`5Xr?Ak+vvfr;|5tb6?q=_%-G;mc_|+1Z%T~X>Rt(7#UvpHNxSbjGst-FY zoe3k3_nSF3%o<86z$(!0l81M43l`;gFuh=Km3J=^vJgXocf1I#TgUGgR;d>hM4Ddc z*8>k*$rEQj6i|ZRCMn`=ix_2)GxgCDbr(}aNZ9n ztYmU6HVTF;31|%0E8iUwxvZWQb)_@mlb-UVbE%Z~TYb8jxatYg;%769){E=5VcSw6 zt~{f0c@~vF8rOPo`Zu0p@eR}UUpJ0j3zFfG#aZp*Q|VtYHgoH;oY((kF#!{6+wIMU39%@7MP@lOjw6pAqM6`R2@|<_u6s_T_p}QRrBi@)^l~c=a@% z$QR4J&77;OMyiq+(&JUzE6$FyNBS#9W#X)dj64Lv3()7Rk60+10LV12)7py*bDW}9 zkIcB+GKwBni4tiw@TJ#E5yDnn(5>Fzc~JEq^T$%sFK4^(b-aRwuG+~)Hn~*YbkF=H zq1)0F;^b5Ee(6469QqY2H&JJe6f4e+hdz_Vpqb8O^ZT`7`{4rj%m=#l5>cFD(oGWP zob=(E#(ZAbi*c-rLjiba1pkwdaMnx_EzPm2Xff|>Ix zsRF$Irs|^W&zeU(yb?{%OPMl#f7oINCusJa`79U#w*Wo90z+5n>&0QYaEqU?q$mqb z2_m))HT~8iukQAh?AYDclx%P3hlQoL>HnX~fC`LQmgy6aLzRhk9&+|pG&H7_UAoqmnejBmsx&c)>P5zKUw$ne;UDiZcP<%j+xOY));tfft#O&# z&ZPRG;wq%KyA`!3oC(uv*c#Vv_7!R3=8krR&khXb4~*O1c59;PblWSeT=;K%^Bg}* z8X5bBOGwmC$uNhW#$LUtYb_yVs;A`+w^pQ%0(=C?wfS>qh0?dmkR`I<-`Q7JmDRcV zxAr~~X~oGmFO{54{juq}lbdk=tY`Ss1|;m|qLQA_pT{W;2(q;Pkf1L06QIevT4veMN8#=b>eL+U;PVm+qd4#P(q_)nbw%dDAvH z$4Euh;faX-NeN% zX!Y7U8XaaGEgkL->g++FC#;V$tBc$1rf0ZbPT8Q#+_SFd#pB=oxzOXk;C9+*3~nV1 zRbA+1ly}mf*G;RZf&2ArE{=}tUapP!HXopY0?(*bjc>mrB+~R%MQdX!HKSa*E#tBO z18m-h#H=QCu6rB#13b?>?-+I|Yos%$>Pm#uOK5Xz|C+uqL*hPgOE~x567myBpto@o(s;uLYYm{Yac=c9~I3tjjK5vnZolBG$5BTqW&$q^%7u z!k*!@*jwIeY`OEHw`Rp2$57R;bhvZJtGBL?mE&8{Gv#L4rx(dD4(q&jfA9g&CocaR|+~|K=xD~{+ z2*>FZnV*@RNImQI-oL3-A@r^x6)_i~>!xps-_H9qmC$k_b|C1@yiW4-GtSA1K1-RU zUq1K_&bz&clNz0w!?vg`8t=IS3Ib*hzGr^d@r|3 z;q^wTcD>iCfu3FeUIuf}uE{6IR3gLGHvSLvNQSMHKi|jqIRLpFo z-MicM3x4JsuPpc<+p_41+}3uOiXH1u4slijIEt+8S8t4uixz*Mk8(QMko4agToS@-Nn{ z{#J&${xS+GB9UA0kJ;|kLw)j0ne&=(X%QJmrkc{ zEH|TtMVH?jFN^bnAWl=jay{dF?7632#!r`tww9Nk#)jt)XYM-I-YHhqMRVl%xJsLt!bM|;nz@@fcY^e~9_Vh&SPW+dnXm+lcJa|AJLg;( z%=>hu+%&+-pkLt~ZJx-cAIq0P(T4_1-}=mr{eeD{01nCLavJU4sQzcu3;mv1lDgY! z&JQgMZ*1((rydU*5mvAGnBQRe8IG?n_FTNOnsZ4s^)ggh2$9_ zit4V{%m7zhlJE%>!A>5!E(_nRuOeOC_6dt0Su>un*y%Hk0A3TrmSF9jPf}HuADQPv zG)b3z7#R4Xs~}f}Y9Jel6Sb>9&%|F}E3x)`AtP}hZ+>+llX=ra{oWLv?9|*eKodzf zhgOGdUfP2T)xDXlea6h&tBAKmIh$>)O=9z-#n;fTaP}{! z)T~HJyxVleHD5|>a;Nr8a{d!oZt7gRmI)py?oWLQW2}tHA#QoHvIV@@cf?gb5%;=` zU4QY-sE190Bt?SFPHAuiilOSolI7xy>~y~s^snSjmU%XP9d`Ysk}wd!V7Ry2zTW(8 zf{RW!PIdefzDj>4bGUnxi^5M%s)C5jNd|{yWPAND&J&_W>abh}_1&?wnJ;F+E9C-A zf}Z})tI00N6)-nO7@$ms6^siKJ%zucs?ii{1)TPiAYHd1c*+1pp?cO>UVqV=S z=r-@57gCgOx!0fYpR1M5T1FL50dbx^XU?{-|9tS)Rp+X?ZQG#Bb%os*m>960xf++2 z2gTOw!G9QbWt}Z=E?xBY47t~QVXp+OtE%Gpm(hkVXUcSZ_7=C)nG6qc0_R4)`{d`xKi+=biG zj_2L{;dK^A%hx-W&bv9S7VRs+=%wm~hR%kI8_O%q{JJo8)@R|rEd_Na8N*q`Ij%Yv zREv~tPdFA%B-|0K$*AscZt%Ahm$2dNX3w(O{8J-Gj^rbXe8FyI zt4Wt9T)r~PWdy$(M$Kn5y`9H`nP!~~x5P2H-n?7TJE_*(B;ss1=+r1{jUBb3qMjX3 zI%kFbu5=FH`F`;&Eh3{q83mj{W;>%FO7?KVHD`ZST_ZSIZM$5We|6akPrXuocg14# zgqJ6GU_`SmHk0jku?QJMuik9$>|DzhudRtA`@KAe+j*Ia-->cpUdG+45;duQ*FS_} zD1Rm*WM@$#Y`n{7AXu+Eq|r8_r>ci{5KkGH6wbc2r7BXU>1=D0uIBLeCWS{8%FfpJ zwylM7Z}-JptCl!;45%smFrToB5~mmVW>?MU)7&NdxS5r!`7=(pu2F3zsV5BEVJfnZ zm0)|+R)0~0Cp8=#!{`DDOl3H=4MrMmU3Xp0o>zcRu!~WS%g7rTliB*7!8VQ^eJTTr zq52o4Bv@|d`(AZKz9qzIi$HOC_WC#ZV8U{)!L9QbBfPjrn}}kvhn!d34K?OWF@M_* zJIJ41BYPS^B#L41M8WX4P(9szwgiv=?{C!}3_f#CQ*^6j@+x$bt8CRDurjl`?sFUT zSG4br^+obwrvmZ3GcrM9a_6k#jR?OR?UUlcJ5D9>$}$48&8{l+aSece~tRahT_hGNsOm!4-{hsr#Fz@u%u1r=yN}AOKAlK zAn%ljhW@a689mZP{4uy|$no1MGrb+5m@iYxynduv%LzTXrLg*Sd*M<(NFdAl{(kAS zDjb=o27kMe{0>RWScEp_3diHRkd`%SfczIP~L;yIjxw-F| zk`{vaH@?|@zPD5!3lJlm6KvB`&bB!k`TU{Qq?x^1j9Kuaa56SMOeNJ*x(`X7v^r>g zGd6VPo}heVn+au!6mIhttTSV&8cn3xVVHA!vTD$Vc(+}~&{EF(VaP>dWu?ScU`W-P zm32>B;$lhv!LEUV#F&1=fmOn&FnD|E%-OSeCsr!)m!4%|%+2@CjQ}jbPLG!PbLMAy z9n4U<_eULg69r_R6G5BqTHOklu1rHYZ_P6F>B14Ov|?vFmJSw;A(&V5Yru_NIKY!| zwLNu!iWr`EWP0DfHzNYQh(-Y^3sO*#(-ZQC*DjM3Q~yqKDBhSJIUjWTZ2kzz^EG<< zkMs_u4k&wW<2a(FZ5T)3A;EXfKVtWD(+Rhr#oXkO3L8UDqmvL>Y~!d>O6yq?wb0#BFM&M=fpOKHVPmo9jm<^WVCLH1)V}Z3GN5F4nr9g|~#x+XlaDp`nQU|6YS`7|f z9@S;Mku`J($NHXfMqxU9cPhjN=j8Tfpq z!=SPA&=@Lc_T(dszFU+)6AU9o#`9;dEe$L<a>lKClo;f@1u<~SX9 zc2Y=UJ6NSfcVJ$%w{orQQR+xt{(6H-)B<3c>+;FfJmyqn{y-)Xdr$SpnFOw+>i{ZJc(cSi|?MNlFMG`BgQtz}H9#q=gy1yML zHmV((TB9!6*zlDup#~&_DUsxDcrojMfod9WqD$F01;1C#2_ zT?qeu2J&ZNc)%Nx0XALF8G(luZ}aFNB1TUwM})k~t}(6oJ}qEOmYBbQNKjarwC+RN z&F^UH71!dS=WONrgO|?o3rUk(HSbY)6aj?tC$&AwbwX6tRg-T%yQUSRYohzPPwLkr zSLEE^ajpNfm5Va)=ocmp@UYmAijx${{BBVem-gkXl{^y6CRy&)j0F)n1)xL1pl>&a}>F+DxwL z9~WDax7%?Z(d?|*VS(wJ;OhALAHFr+8+0c$ZZ2)Sl#AAN#lahqoM`JOx14k>pTl2Uc}QeZL>X77W~7 zdc5E*p67%! zYu+=XhR46OsUllj{AljbL79O!JE7MD#HGcM7t8I}XPvbusf$j3INO%&e-0%Y9df@G zgHA!N7y3jEATJHuZq5K(UA%l9Ew+ z8x6iaO5rrDdM{~sR&;YVba32r%+n&spE^INg}Z)kyE#dcT4BCB~S)pJ|)k7>6*p~KA^eW3P1RStT!Gia05x4rPfo->I*r)TwY@2J=?KJ{^^W2e$P(BhysTMOiE(oYff|m9Yp2&*QJ|?&tKh%M z2Jj9&Fu}*dv61BKr;>J~Rc`hvj4f?D>(x<*y^HVK0(ErAKHJST=t{B~u$Pnje%f`r zmOJ7o(t%%wrnghf>N&u-id(smdo4nB<>u7_>TUM@na2z9L>LT5p3&u^Vq-YxPs^Ia z4$FqZ#8#%g1VPz8hgWzdh8QB?As>)mke^`5quW!M0Ko@8%-OY@Nb7O`lZNK4sj=nu z4bgCDla${S%M4GpA5F(uVEgguSfRvWY9p&FJ-j(fQy+jFlzD+XhrA!M;rFhloF!+y z-JZ{QM*CP$p`bo;mqaK0_>B_*I?e^jxOrKh2j~T5zo^B$DC7XAT;Rrgy*Y~Z{FLDR z+|OTEHw^{OJ*FdDKrS(Iwaz-PReaVEvBgs{>*Ct}r$#n|!rW7T$LH{6>8Ib7rA*$C zR$p$KvcTh-&AQ1+=6~>~`;9X*5j%Eic5rq5As0izrAF>ihRXW+$uhTdavjKI&X*+* zn@HC=nuY5Ou06K)GIYdJ_2K{p_Ju!qwfNo_bJp7suy$S1Am&@q+S98Nkxm4wwq3l? zP|MgM(f0f%aMuhDszsV#s9bxVq2lg(tlb=_s*raq2+usPsN-O&etzmA{$$kDk}ZZD zb)(A+^mhr6vA;B|kuvt(VZOYDu8ef@*UWCsH$&!^4+u)|OTgD#LTMi7;sZYRl`Gex zsy_f@>gIJ2@M6a`_XDIyeV^vH0MG)ZD6o67TNjS3;Uv#i>h_ww)|_+jWXtZzay zJh?V%jzy}2aI*m(G~|}L>){*I7pei0W!w#QF$TU}_ut)y+X$|6^4$DU8L_>K=p(Nq zj}~%iC@KU=pCX7&U(=N;(Za9u!SgAES_5qs?pF`2wbsrtA|qvkemhY$j6x)OnM@ z(;w_Y`^0Y>`P<7)jY4b>0A@>y#^OtP$H{Xp%i%+j%@&j$jab{3#;F_<+z195QZdDo zWUlW6`)Sl}txOFd6F=y%*cOn){Tb+g+miUJ$^T5r4mt>=Y(};GJ#Xx)j)T#I*>e^L zoJT)El|I(*163;e)(g4$#Gf%K-(Y)ZfglfoIMKM>`c_o;Ql`!KVn<2dv$VP;lW3XL zxBnD-6R1Ba*;5F-o?%1-@Sjlw%C9@N>+q-Tj1enF9Qo~cmEji38CrY}3s#L+@h(Ax zr67{ee`p(m`-T(iSMIxi#Eeq5R%)kz^Pj}cb(?JUL0ZaJC9`y~fgdPoRSLtdKobt)E)YT#Z|Dj$eYY>>#EChM%DP1NiA+SH{rMwLFDV zXmc-g(S1K2vxjPHe+&x!eq{iJ+YG2n7Xr2sBme{S?pdFTn?AJmwu&tXbQ`?9ABo3F z3jL)QnO0+FUj_^3ysPju(>l<=RR~qIF(S8pfD$_@U1C7&DxZ>7xb&%dZ3c!O|BBr4 zVJUFXK`LH1 zaz;c=fb-8U`@mY4JlA_eLua2^S+%eGZo9(1BXh`55*zvgB?F>oA|JBA=3g{g_P>bR zN;kbxkZW89-a^&Gb22*ixxYP2lGCkbE>g<1dfaeWBsY0q$KKnD$}m3Mf<<%b*AH-Q zSWrNpNv)JMsBf##*zNKdOm%tvIW`$`CRn3^0@jW5_EYFLCtYyD>7M2GfJh-Lwi5N^ zr53oCI%#}tI9F&Tfx?B3+%Rat$x~n6=w5t!tgt*x>>$p5%j7{Z1_;XBQL0C~l0cvo z`yUz^q$kx}cjDa?lyTycgV=F|da}L(5b@=;(#U+*|Z~E1G zQ2kK98po%ebdGxcP)Z;R*m2ozp9I6af3_v%>0NRv{M~!N(Rc`iXe>Nfd9yDeAip3~ zdUj>GsM=^Zujd7art=K!Qvbxheo!)Bry1t=p5frG&Mrm?w?uAVgt-^To2G9aQjf|2 zc94X-w#`6ZDlofJ9G^1kKO2fe@SR0>Ez<5We8`Fb`aev7>iU2pbg>EUK)^==zmVl~ z5KK{)+}NE0fZQ)t4%g{iLlnv;r9AaU;7hkOUgum9jM0p+&Ee?!T4=5=CL&GMP#38E@rdN=EC9_MWum_8M3afiQ$)%$IzIZ z*VlhkKY+w2I~N`L$r6a9@^YZORzs}WiFko`{~M7m%<{b&;M)r2npjk-U6Rj-U&Nb({& zE&kRP$)OVM2f4p1d|lRlsj7P{6E~J$OY`{3ApqBfHV(I%PVi2%kdY_}PM5{-g@y=k za1gl}l~LE9{c>t+fZT|vh#LMF)))X3)G-M~rQ6)FE|=dlVoq)SW+XiFA_5Q5-`_zg z-_r4G!Sw5v*x>($Z+u7liO%LLI8LXo{I0vkoM!?ij%S3Vb|=oYhQAWkRF_;uHo^|- zrS6ku>c?VR>VMW(G~_D$#{O&-cu=}58mh1lZ<~PpmD@^s^s##Ei8`ecJ`97;cjlJ!+ z4Lh}6=@Dvpri;2}W3$hq1Qy{#&9M3=o~G>9Z*yhd4;s1cZcUEl(;6qw@eRN}I>lS0 zMSGjE$ur5KX18Uzwa1Z=tse**B1D{_-q1aXzT!q}aj+pcm?X%F=+vNC(yZ)U>|nKk zPo%N5lBtnabjW)YjxhH>>Xk0;^#U5^0)ev#^Yf_O9CJn4&!6$v13t{?VE!U zTXjc_%q)VZ`5lHB=Eh2R;Hx7`K8yZTi8Bg&oIqrH`)vQ7Y;S7y2RE%f!2K|9zXjEC zdRBCRD&S(Y4_|coL`WGikk#+r#^eKO=}1t_fqTo+?W(d_Z*_TI0T$J)L+nJb{Lr;h zGG+Y{@Ma`X!J~dXUwhjGKL^t5{P!mzx*J=^r$fO;gZ$rBf93-ZD}ivEvQ~Nl8-PbN z8pmp>!q0yj+6{z4ddd4QB*44sc}sPBV*NBkTZ3_ZZpb;%Rc_;9CUg@u5S89UEUeR`o3*s|wUgqFneAnb72_pF5cOUX}cc z&5I&BM-(Tg1|nn5XJkLa07nwR??=x!U~%BFpOvIHf_D{IzT0gHSa`#(6I$ab;2jCLQf z9p3XAows5&-tu*{#X`HWR4ca%t`0luMDQj|${#RCSG>mC5Wz{?b69WI*>)CC(^V#! z+o-DzPut4c@m@SCHJARVWL@PmTgKg0hJj;rV?EXymnpj^9ps<6<@?8Uq^FXgS_+{K zVOF!7)a{R!!M%cu_R}L4Rrrpo-E!Qu|AFcP)NwU~Q)~wA*3-P(Y6M!*d{G&Mn*c!Q zXD#l}h8*~8#;qoP4LP|=RX5|eZfO0PZcJFoR!q^KO_`6P)Q&-)>T(0ys`+3c1R&t6 zM&%~IDqroJWm<`Gp!>(p7^wb8lPvQ_>DM!pi-t9{5tEfB^WxpwbC`W~dN2V?u*8an zHay_Gf{`d;9?wFz>U$#@U2Pwow*jSsK(dR^w-=RrA8#e|05GAx(%7dXW9GWu;#4IC zRS?BSG-M_UEib?-!w~Xa*1i@4-I?)$wUCht5;uN?-w;hdKP4?Kc>GAbdvC*eP_8tN zfD6xcA>n~xSB(3V+;9We=-t-(0|{og-iUn)*iW}&69@fbfXz@2kJi>SLv#?_FE2*) z?hS?0L0;8L$B}(m^WCj>a{GNWX9FnY=5~>%7h){t8fhTWAgb;xG1zvmWr4rX)qbab zdw$ReJx!)MNM%t;-)nqQerH(s@ehfY)e>*Dfy9cw$?X_j`+!3+I5xV^!{x~5A$AL{^(6h>_Zi}&M^ zFDHxPUiM>YVT;vtgK=pmbV6M=t!;uBpjODJG=Q53R};dsoZ^>+tV34jx4ld2WFW0T zC(Hf>0Z_(et#c1ecNQr^yd;@vxXN#>m+iWUOE^fmoFAhqywJ;)M7ed>h!fxYys`dL zk5NIjDbZnSp9oj013%0seFJ*n+cH?qK$d=32xE!cEvJF&Sy8X8s~U28e&Widz_ABv z(TtvHy4R%l`Ui$yr|Dx@JHc&=s>$BU=U0z$!li*U{-2ETcaj%o87-$%7VXy>E6rl+ zt|i>mgH;1LVUeS+9Q?Qx%BufiWv)$fA+)#IZ!KkTQiMU!h?}rdn(k|8MKyKK_ecNM zU9W?HIUVnytoBN{{6X{V~LH zWoYD|g}i4#T#uBGSU{#M5`Jpv=F#3bi_;6n&dlNuKbC8-662;JUgS5 zyK%h1VsXyprPIi8h2Bu<$S=KvbDhC9=QA0VkyWF`BxLD5KcKTHJII}7EqAsq;^Wv>852?iZ zr|rxM`)N#u@lC1pS48`m=aRPj6eZ?=ocqZQ-Q)P}X5Vw%6~)SpZ4Go@lMi0GA+~m4 z+7yA8KONeH1`o-n=S|-_+9_P7J5zucAHHInxp)7)7XF#LGSgY+C$WLh{OvnapgT{u z-90nWSNFL?`&He|>8(wgq(hTa+X?Z%VlSW4yCX{q7`Wk5*;TTwy0Y--3#&&G&~}r- zGf`&?d){>J^uaves=|;e`~)`1n&}=-T)>h!sUk`}E|V@$PN)w&H^WW&p1dWFXH4Sg zo0ZS$_yhOt8wwxsa@dZ6CPr!3w}JC#K0SyWNH7Q};XI|9%J&O*@0E@L-`y#|y!L3k zL=o22(a~4GLZS{>f%;c5K+_+vT*{WrX(`sIZAC>y&j| z<|0Ab;MfBh6B<6l;HGizK(~o_p-+PWMg@NLi6>h-42)Y9-DW@+1LZ(+6xC)1S?ICi znQWD_^&PFB>$)y{AON@e`!X8#K)aYnzhu;%k8XSG&v(uiYJBKseNIQ9&QPq4SlWL+Lp zOXw1#V=Xm1Pm+%+J|ADMGq1imzO6rANQmig=8-ak8FwK;_O#4C2c%cSgILH|c@wi8 zAR{e~Cy1P8YP`vPB2w+6EpicX8(Hx_27tr1>ZNA;t0PvDo8eLnWC+1$_sg1x6#qcO zC_i)g`xJdQCw%2FvGHwcb2JC&NP|cBmTV!Z1P7|d+7baMm<#AhIimeAF>Vc;)*$gP z!JbHKiOR&vYns(<=lNartBZAMWATbihz^agc?>Db1{Na{&@y)J+ z7EGXJ8noKcuvz9BW8YHgXyH@(M$1meSKnO-CvQfy+)EoyhHY)+UzgvS1M&m#Avx8ocBMk-~~V%5!~26B@(FWa1^`CzBNDoaZjPt z1ga>O*_>3&Uj$IMiekG6>nTP1KZ{D-&Af7iBA)jWlH?$$T_TU7 z)V?Q?^Xhs$0%1m@8yy8 zuDqG{`yeuz_ixb19;;W7?6&qMcc*qgi^B9dSbjt5gAvEF<+oiDJ5*KS;c?JeHGBzg zt$+`won&HoUM{D-6vVKrQH6yE?T0e5H{WUKYD;}jo8JufX7^YC55bv3 z9;EXbbu*jJ)I#SxHIoFE^Qr`zK2#}kxv26GSPfN_X)mECS{sdSw}({tl#^EM>OHAj z`r3J|b@>MD%7HXm(EDI7!7d_F%juzoH+x$3+C=kudUD>?-g{zIt6Tr(PK`5KDq88p zPXj1y8BrvYZe7RhJ|9nBbl_di5 z6X{kcwp3Tb+{UTRt3u(`IDhHf2Js>!H9ip| ztcwppDqHKXnxEw@E10LLvy+v$HI^{d_>3NRr)0K|r_;!}Cz(j|;5Ue=O<(y*Uu(#_ z4!mQoG@FX%@B69O`Y0#`D{noVgZS;#c;W7tBZA5IYFfK3mR@K zT9%n~+9^8^^`}}B6@OxEZ(SLPfMtq2))(0VYxwA_~To z2vbQ!Rx=b+{C+|o%35tw%BwEe_wW-*>{n~Pl+V^*27X10Iq-6emkP7?g|!6EE(kAub%lnp5YsBClop zsy1|!vd0xpJfcAk%75Ne=I_es5UQ*4HQN!5tQibr`gOT_!k-3nzKjo03X(E66=E;? z85FM(Cg3O77Okkm6&s%nL3>c;b11_Js~?TvIPeBr0)#0e4u6Z>J>4N~JJQ+?s$H5J zx};SiF46S1AF9pNJ!>(b5|6>N=7->(=JVW7vIG@`M8jqJ-3S5AiIy`(@a-Yiyv8@|F|^Ib6z_ zFs>MP>6-7Q1GG)TRy3KupHoa?%YMU-hy)ELYDodtPT(G=@hu0pU>t3nOIdHlr=WIu~x9LJt>TV%-Y1TsGnUOR(6e7gVl6TszD@pv-5JWnzIr1AOQ_VQ?D=Q55Gx# zw*h5a^Ix_wUNx)8e&O4y;4sm%VR0Q?iX05qw00!vF3+0133nptz*b2Qv@rG!<;tn9 zWaw5)y|MZH3aiPd&_bnf_hkKuk@Fn+YYkFA^ho&28qJkRZuJJ+-2A<^xAyGEtWWL_ zl|NJz>{<`D;p;ST2)O7``3`P=HyTc~`%xOwz@iV>>rYD2t8(+r)0XKErOqi%Djc_x z`R)7JiD=2`h=9N4C^iyvfQR#3r(9ATOuP@@>3$R|%_}h?Nn(wm;N;`V&$X2AOsZ>m zJLF>de&OZ-OxxM>LReO6fQKUvR==3?d?Tx>(E1aLj%L{JE1wYtu&4*M9D)^4@vMV`wxs#@xyOXnVU)we~toAh%F07yt zZ`5kr=$iQw6SNS{!LNRKULl}wem|T;g2eM(mfOPq!<%fP;&XHT`^$yV#a&@5Wu>Ch z%@lBT%L<-^s6UJ1=S8xr32(dKK8&_|f}R&V(^s#t@n-d$jvuo8dWg>?JwGL*vL{!x zI!x7lQ%`JuXWvR16;NGXll&OE2^6sk%u^Z{J`MvLuo+_2y3!3w8u>{)S|i$!_VaoB zAc;_9tK+Q|Y@zL;&BA?((>GlTXFJnFd!1(zFZ72<76-HxwRl<|OV3wLlH;B3Nww@n zkYpcE^Z$@KiGNYP=E1VJ;*trP`}nE!HO19ie~skIzRHwT*biPwc02Z6a~7Q`c>3UH zr8H(HjCNQ=*kT;{MWyUpbzBhTJcG_u+k2Rg&28roQ-?S`EhGKNKmOybng{=)qlzjI z#uKk|k|4o*HuuEkw!@E1|LDLw9;QmqlA3e?*FA}Rs8H>YHV@{>Y797(idh{#dez{5 zsrU1@L93Qq9CEQjSC-!vhO;#Y4rsaLPdWI0yhc81(0xX*=yKx96-iW^V?2ii;$t#7WqOv^ioPB@O27-7f?Ux>(N8%OrK*zkrvo z?iRT;`0kVJFSn)?oC=v23E9AKAT{>UkCI+k?R9*MJ5c2Hyt-ibhuyupTRLG?rIld* zsOnTuvO{Rt_T~>?zGi*9`ya~WVSNt8{S-D`>b-WGm@>bn9NR`dy`#y9=S=Q&b3VMw z=iIm(-689DsT+kqI%}zmSIPD`?apE_XcXq%aWm*SR_9YPT;u`^BwU^holD0hybA4O)^g zkKT@7>q80BCe_`quTi!^RfP774S6|?WSu;zk|l0`m-z#oCrQ)J^bZb28ZgqUc$#GC zD);4^iUNtZ$~Wf54gFbn8{@bP-f^C=+)4~@k4uBZ1yyVE*APq|qj0xH{|>|tq3p?3 zS#J;dAr?Zt`@Pe?HPYXr-CQ63nG`72%_Nc#Q(rM1q-E)45}pK4(L1%u*D6}QD}TD+ zJNeiR^j?LLL-AFc8<$enI`$qC`FS8WM1Ww2RYiAcC%F4+860A^F}gB&m>)lvn5Jqz zV;zgbrBI<-sRk$>|Gn3NsZ;%LBfr^P;;_@ZN2M7;kIiS@TB&v{Cll974kYs3rWL-m zvAQ4Nk+0CfxsBCYiKfM_@`zq!8xHlZ`viASC;AjxtC{)6OuWKKDwK5EuWHbCEgUs& zi{p4Wb&i=dv485Nq|phfTCXKA!<%Klmhw77zwT7SZ7N2-BO9dcyOX z|EIp}={xADb0lD_juYC*&%z;{Usa?EE{(FUcP>=OiNVK8SFPDb6f#r}Um0#&mM;;I zkL4%Y_61&VgZ3%R9jC_X16xJE#ve?Y+F~u-2=&5&4odu=wh6fXtkSuZxId9!b+cRS zB%cgl({jShyW_{n5@PXpqbj`eKoiwvFSg9Jt#kZsJpv)oKZ(G>&bU#XUhsmfs=durJxBGSTY zw7uMMXW7Nu*|Z-^aOg%I2e#nTI+RqcR;IwN=8TeGNFZuDm{;H<@>`sxbGe^y<1Z z^^x$pb%^!_nfIkV0NnO@)GlvyH+RzOKMn!^sbj;dFn9>8nbttq{%jb@WffpW)3u>inb$4ETRf z$D)85TDCK&wo`hZ8a2N(1F+v0593ERJ<$i=0#|i3Ww=H?7C>a82T((>)U!eeHF#hoj(XaD!$o}s=b>@?I~A9r;rlR}_D-*3 zP7RK%t5gSkq->A#!7Bp{z>TweAZ|4H@2&q`n8`6PmX`1vU=jFMQ0;5_ zdF5!!XSt4+~4zL5YSE zlMb|cuDe+qV`JF762QhXjFjgXQM3+VL7(o+xT;PtQE6S2vVKo)u1Fp(cX#vATiN#8 zZ|gR9#F)w(=6Sgpw_AR&SQZw?NlK6|8)0&p;} zO62dZUZaSI!;~>_^so#ffmFbfn9e|RAK(luBVt$yRJ0- zf7$hLko5cR7fUUUSkTdLzmP7hJ@+x*$amjLC;MtOD!o;v?@wKXYi{th1nVrEEF?qz zkeGBVxSDlk{blzwZp)YGj>aLO$NtCSooe5@E?)@Aq(S+Ae&8kwPoEjDGdFX!fuaMU z)+=dDl(Nt(3-DBPvDM%D5EYbAjmC?;QU037)nWiMu|Q^{v}|MQC&%;d?=3Kqog|Lp_jy8W(lMC@t>`l zpl{0o|9|q86%zK}W6*P0rFL~H8>#H!?&tJa=joCUp;eIEq)Us4- zN6r%cjp@t=aAwhEF{*42I7LhZ?T&Z0UU)Wob@ldQaV^$KPbI5p^p~j%dUY*QCII{C z{ho1Kw_#>ROisl`&2D0iiKjHg>9X^SNdK|8f5Y5Z&X%Pkf$pQl&NJP*b`gifLqXb? z4!PaLC;CqU@{?*s=!BqIQJ4q=sw--@!xZAUIiJIafLEiii4f8u;m4GVm@^-sn02_v zyp>lW^QpbBVV~tmPc!$i96+nKPJr_;^7;nxJet?blg-$+TC5`edgvl8C>LCxgc4nm zL{Ahb)kN|`a~K`t=R&sA06Lp_61mvEyjf=)c~kn!qRVU1l-D1cZ#oMVCcZrLxxnnC z>q$?H5t&HKwmAE(H=)QYaVJa&vVXI7Q+@Hx^T}BnKwHwkwk7bdZDIMi-*b|p)BK4* zupAcjS(~`Cxa4V+E}&QD^c@S!Ue7zv!y^bv>`Ep;(rTwP3*`AjiGrgAwgUQqwq|n8 zReze&9?PP49TcWsX{L%=bPuRLFnGeP^<3$!Km}kg?ath9s}jQ0uw_JYj?FUGGpY(4 zbEy)G7J_-_IzXO9^(#&a_<&Z4I}(=^`!{pxrJhF|?@9)3CQ>m$X&sYJl(@4%bh=(I z74tQzPEhOFM7x+g(Iw^9VB+KirJ5yrFY0&zlE#F}^g_fx09cW9XCde90D3zB*c=G}ihIoknn3ikeG_wgN~LYz8Ms39S>PbWk8VS}cRixKO&aFwWj??5wm=17 z@>Fz88|*FS$k!QAr`^7D@I&A+*N3)vcpXli3y!~+Rz+m%+jEVP^k|7(Xzl|FZ_Rq+bebb%iPViFJx5)zV^pfwI< zf*aL2>WOx$*S-cCyb16JfL4;3(?92Xq@Zdj{d?!25^*m3G4I%1smtN`pVOUb!pjSh zU!I5aRil8G&QehD0Qw9ZYWsxl&kRPe>YjRaN7la`XgFTys)Kn28z1znn!4upCXFcR zeuS}}Jtx)Q7rx34=J z#%M)ys6PB{e(CXggfr$uUc7*0-5BI5S9W^wlWh|j=0QTaYuy0eU&H+lb-wu0pVRzdoQBP|oY1Fy_O(iCKr9ar|Ie zo|LkYs0VH!Oh(vey@XeCa)d%wo?ztE9NoOt1k{UihbsFmAj`nj%r|Xb1W#2cB7$ZG z9kVEco@RQlww~Ylh#6=fa8paL4f?eE9{-l3-*)SCwW<0-tY;@Fu0KFEvMU^{im zEvdz*VHi|n!b<1d>_UY$RB(jq*$s8Y!QtCeDhV zyj%%3HuLfQF!AUTh7pfu;7uH(Yx(?>c|WCQ1)2%t`O z{)}M_tka(rA8@iLo{L}nv}9F2bPxBNc#h?Bx;yYnnM#TP`;1n(wkN2zLx*g9XQJgl zwEzOr<}2)H`cOME_dcH{lh`Tx(HgBp=UFPV4pBrJl}{ho-mq`iN!ICnhq-2dfvT%D z*u>nd_`Hq>BH7zHVCHCZ?@%NNh$B^DE?$1x!Krg+w;X77JU1%Wr=oI~@mG-BStUGk ziZmPlw@3ukG`{>N$-8@68s5Gk%pNjNLeJAp5$a`&Rd{Q-rjSL)LXAB`l3(E*k72v4 zupnsAsv-J!U)pQ|S%Cq90HChmsjnkT_Ormok-dr9c!7UN0o0O zQ8v~QPb#`_Hx~;NcvUFdyWA%QA*BjQqDZ79I&Ll%{A%NRT*TTowb|qr<@-~<()oP1 zPIRL0FNeHRIo-Az4pqBrWUX2$&yoydfp~QP*2Ob5)1FJ!uiE1T6vgdn;I>^2Nj5wn zN(>x4^+qaAce&B;1t;ORp!lA>+OiqJz!1ms`G_?kuv~i4Hn)zZ2(ny3xfk#`yBmq_ zE2u}PW+%d4=u3fbPBG;huGhm?2tT*YnxfJEp30Yy!u|w$@96@}Bd~&j@*Ws?CpuIU z7X{zwBtDvD^5khcrK}#4d$#_x7h2iXpU%;4)y>ReHg@|g#Lz41WNdu`h(_h7E96vs z@ZtuPHaariG=#KEUoejNCAd_XsOS!LGv=3js{Ur@_FD^BuiqR5J^UuHYxK;u$nzz@ zERkfC#}gj?_ecnC6caiy8PriT5OzVUMa+a9w50ubpCA7`i~8~er&pv2hoQxDV7;+` z$y4=73H$0~vpOmZM2Popmz&KwXYR8qiXpfu18Qwvcl^GYS6?TWX|P32$C>DY72iN4 zYKYJnvFW!}5+DC^QF?p$jY%gVwg!a!$eHo-?b6uW9MD9KA8SH;@ox6NLgwyQRbjto zGCW?+^zAFl^PND8+uCp7Wl*q0yyOapK79bW@7 zs`ir)|60;NWlR7I%kahM(s9{dXS-1-Jru0PM$#929GTYko`a0Jaxa(Em>AA*OC{HH zhkXq_{^DZ6N4m#YTAJS3lt{(FhDK7cmlry3!Ra9wCRykK^^hyza-( zbaZt3Yko~Be*u_Q@Pwet_pPFGcflpvTnn%IX;~I^9j#k>?d0O`lRhG1K7V0uTCotU z(#1TLdPy8=iGSuc@IkGWl;&$2*Wd`gLm)4<&I!tUJW89#sbfjx0TbXt+tHUW*J9fa zDJ7-j#7#%la$VIRHQ2yV!hsp!%&f zl3Uh5Zrf$?X;g*9o$J3>!(&{>uQhf;d>bqENBn&%V4u}Hd-c^kOn;?X&-wfgX}}ZH z9+qJYr|eyf<$8t21*~TKMJE6@NIv}Wz0P6dc||pVsddu<-tlw}{3HjvKK_9QT>sgTofEwf@a ziv0Xptut|hr}D(Uro(+61s^qrqYOe{`d@>P)7ah36f%k{r{kD9H?!=6jurVV`yVY# zq)ZmSO%)raBy0*Z_#4)j_zgvzS>R!e2FLB^bq;f3K=iWKAw{H)!t;BU_cH9+PGl&e zo}Ayckeh`^lR`B9oMczfCvp{fm9b#-5z=&}c9`{~U5T89BKWoAyr`+D!UL?)=Bg|r zU=s<9s|iPm)%R5{LaKC9!Nug|UnO2>l?8|r?5u;A@4~TE5LAi8D)VHZr{yx)92kY`BHKGI5K<`HVe;4jG zI3MHe8#K}@Cl$TNq!ZzPGgR&k6jWGvCpk%ZcuKWkxs#n;MMHwJM(w}el+98-;^9Z3 z1XzrpFtC6#=mBJvIhD9dStosC3q56Rh3&XH^ostRPU1r%Iz+&n-$3EC>!>>A3317v zmCjF~%P_i-A6Ms6L~H8R>HX`p3x%&Zd=J~?`4&A6WIYr-jwq5OSDx*Jf8F9AoB6aF z)5SXO_Blg%DmAB-SlBJHUp)FtRB!~8n$xJ+@ZfVl@!^2y;Jnb@Qm|~_*!pfh;^UZ2 z(omjt@WMp_g4}ApyCXZ&%9{{Jw+gLHlA-hLJ-LEs6y4|;nzup9ZBfq^|i(KDw%quQUSaFqrFC|%+o&U_rCU@KCzLaL#LOfgf zAh*`_r_b}Q6W%3Pyogo6DSnAN&_Hwfw76U@@Jgbc=X=f)aL}b-Q8SPeuBN@)X%=! zNWZFhVH^HsYI&kkUm@-#8%-Q>98h#Y=HqeC&{m>!=~Bd|qT5$Zj-4ZufYqt0g*(eM zoWPs44wetJnJFVb{PEp)iV?`L4Iv6%Wz$Z(GQ-p3_Ui=j{iE@ZQ0pE_WD^t z+=n0m+>;?oRaK-^-YUQC31*n`QO`9Q2jYoTm6YW-#X;mNFYu7F1hh+{!Qci z#DbCym6gjJ^91jIQ?uC$(cc_9p?$uQzHOY@lZqS|od-QLFlK)qGUm2F=)~zd2A_xeAAT8aE z($do1NT+mnx1@s7UDDm%2-4jkN_RJBzUcbax4wPOzOMc6{Mi4L_vM-6nRCoJ$GFFR z54tam4m;f0UH5T;1AkArKj`nVU&AzZv`7)R6%wWZ95&OSmZ1mH6pC$xD4E=))QY)& z>>)-Oo85~3k^-=%Xt6Q$D&b-IYzCXe!F!tCqeE$?fp)4dXW7iUD-K3GT;7SLio?#) zX*qi5+&z0%K}E5x@2~BC(T4FokwsoV_ns#Ky+=s@j8;=&;MmSKlb(4fQ2Z6WP`Q%i zNm*w|Sp=|ax?A&G60s&1x-O{!&7|2if`-7C)+T+*r0RbU^{;eTiMEx_-zu(-%CyGA ztnP%Y>7H@))1zm-7AGxCG73l)%E908;JIrupv9EJr_G))xuw)F3`FqOoUQpCq5k+4 zC30SH^$Kg38|RCXlCG-Kqls$zB!SqNgdVvV<`>mFgqqJNG$d~KW@}=xXp$J8Hd|^E zckBj-Ok{RR4J31QA46&oE{xyI@jrT98!QgL)H@$FJSp`UmT0ilRwp!(-riKC(v^=T zc&}Xia_9w>N>mGOi7_j`vU`fj*SBsci2H9+=ypbLl2w}y;9u7bpl;bydedQ8!Mo2R zAiB@%-0ZRwQp0SGJ}=VE6nPMyQcJQG?_ja`I0QN;Ro~iG^=O=2RLd}R@YGDr&w&^$*=MP`bv`p>gWR&1IA+oX{cd(>3V__2iK}c zLe5q%qABr9jFsvRmqxZN<6l``XjI&2%}NZ?&?ZUgo)`Gv67sz+w>~@7vclW_uoa`2 zbvbcUqqMhu(!6&~+nb*gG#<^zt^ZSdtUw*{%NJNd=N2RB&7NnLS}u@u7>=wyiRP#Wu4pb<$E>@BEb~^FXGM~p-;dM zKKs>bqENgDv*H_*NsU6+f1B+b#Xp&DawQc9_wka`GM>Pz9}!x}(^sU<}Vf69!&&r}ezdhVQ-(IXmSfr1qof=)=V|y|m z5FwP@V8Z->`6$6fr8ueZGHIvW1%A_GWQL2*iH;i#J$rdR13M~|edM~A%M^t4v_MqDLS{`5B+w=}J^0og%97K zc8KDcwbR&8CdpVqt1k0BL|j6+$Nw!&16JGyJG@f4x5!Of$pGjDX3~H0mZmQH;)RJY z8+Hu!Pg5Ix2WHpf0<$D4CEgIpj#JLzA7*y9c@7uDX>NROH{dUsPSl;B&9ti=-*O+B zorNOr?rH~Gnto;K%^f_C$C2IVN^TmgnLunq2jFAR=li^B`*&XTeV9NC6NaW`ZSZu6{UNKkijo~kpH_G(Tt&HSV7cKgPtT@fpI*UZVPI<~iJbF=)Cg)Sh zqL;LNihs=7Si-9+wQgGqkT%BoicWLA!*YS=aJj*+?NCA$)HqKmS0C^NCe@5Q?W&e3{uO z`0t5~eWL^zS{(VoM;;P{Yv?i@vx-u}48emU_?@k5KdI+}oDzrx+1tvqJ8dRJVNDM# zp&vX??$q1-7PH|Z9c5hq)j(E(KK*iD%GMKbv1J2C4AaH0m`58B2Umt6R8Ut7%d2#l z7cmrs+nNBq#cwv?Nm->NRdFmz-6Vfk<3*K!J=0fh#e2%Hw`ZbwE=@q&Hng$0FMuA8jyI#-Nw3qoRqcVH4NwO0)3SMN( zuD89;#v|#$!U@OlIJqs7sRs0I#x!Tul9UoqF>wQ6 zy%9T=U7xJ`2al2)_Gl{4a`Bo%Do3n|VU2y>1XV#zqbwB6aZDNZ*b11giY+YH4TcDL z6Xsr}P$7b%RvQcoCA-WHXmw8r$jd&kJ~_c3A4_qW8hh}J;_LguNT$q(EC`EH?GqVN z%>8j@$49DER(%E;V0BLv8$0UA;5M%=&F7M4Ox2~hvKU9sI$_zb_7e+;K;S=Nwfc?) zU4~wd+ua8}yYBU7SjcwmU~Hg2b#%-EFGA18WS-7Utd?J{ zG4p@XQIm;1KMcB^NByx9PvrW}{K$cOZTq1^ac1c6c5T!6<3CC+XBs z;v_H?nl#a2?`^`>E0qQKzBo=Hw3m|gXWSrEjfCFWR67O`FObxb1#FG#M_RxRDTsF>Cf*L z21|>3;HpbX-WpwPtmMRU%|MHw;E(67Rf6-a!t6?!o3nLvQC`o%{=-VKW|I#{9U-Je ztxwb?rapD1vvd-;?4oXCdFJBZ)mX$$4Qrdu0w=K{&`j5I{J2u?+(`hc;`9WJOS9nhXU&tfJ)!>C}sLZdA@vVgU883 zTaAF_?N;87n4gcgDYtT!PeXMYg^DeDKM`y*qGQ4$nXaVm?C!61fkH*)!Bm;S4$RA! z8B6PI(CUu49{93@Mrfobc$8T!CO4l5rH^;7ojK$|7^i^dCh(R1gQ+ueq#>+Pv`%7V zEov#E+}*0a^YO^Y{t$6Hd5SLkgS%Y9&V3KLSnuHs*rSDEsGb7= zUgPlA;`ky+>pH%8hV(7IKZwR)b%NHANX+gPwc?xTc`3?5Nze^@T(5mZJi}w>qpZC?lOXN+276nW0Jv1E zkD=o2Xj-{Fe{>q~7}Uyd18;v!hhT9VBfX8J(=2<9Sr8wghFm$)KU<|!k9uk%5zib^=3}p+f$P)YQ(`?%)>s@)6n7-|#pYsbhk~BvF@)~m!`c@n25b86idH^% z+ZkQt1Ad=EQT<@ex1I#$lLzE@U{R52d3{ZLjVV7W7ZQtZi!M3&DwpF?=iF?m8sZ-%5>m`5mE<1|Ez)y&QQ)VINANu4vq$n& zqea}p8Q&z0N$#Qo^4#z!Z+oaiW6}?QRbddi(FpY8m=LCBmWXDR!PlP$LF}3BHitPh zaYAMgEU~yVkuT6zVPyin+wLY*qLEu}@+dFA$56~YFSMB{(T%FMniJ;F95UwVC zuwv_CZ>Z*Y?(rph)LRO@WhLdS&7)k;gUARo`Et|Tz=JAI%O$kimq{kzIwkY`Sunab zH^Pc-kRd;@KXFPbb2Kmh;ORl0$0X-o_KxHHpUkVqK$CXqE1`VV0rc$>^mODC00Nc0Hh?f<*yJ4#}EyZhgJrqJbzM>xqv^O$+J%~ENxf;D^&B}H^W?OQQU(LQZ9Xzp8rZ5&<@igo zZ^*+E{h(b@dS~QENk9ZNP%^;@0}gZ8Ewj$zsZ~JmFC89M!DEmN%6p<#-A!?KK_Sn3II_!1BTJmYRxi`%l zct(I{AZ&z*2&ft%469QDGAU~x@ykmbwk)jS+?fj&1-o0^SKWy6DZb2WlA}3$7{?$?;_1#;^^jTp(KV ze+-ZvHK-snAB3Mw4>_A@yx}?(G3r7;>J%cZRlxZ({v^iv*_Y8By`qeKoNd*()NIGy* zdkXH)M4dtucg=Pf8+r(c7|wkUBEGa0J1J3mZg^tvIUixVJk&$fZ{yD7_RCOZRQL}@ z&i8}+BoJ8@GZZD?J$AGGBX5LuVn)chTVl5g5;i+aKS3E2XFi%wr3+>QkJ*!_1+{^d zJROpzpwOF*f#TjctbG6cc`bRRY8G|14J6aUqw9~*`;MC-v9Dy)7f(C<^JR;AMRG-6 zyBZTb6h=l(Ide=1uxRhjepNsfmPuMG&}DKgj!p@NqIiCs z$xrCbq_=#Bgn{@=zz};CyHtU{M5&sH++x%A7~_7vDx9ByG6*Fk9yq6iFtWhriHVK% z11y*lyQaK|qY}n;2A3bk)1E=1*39IH6C+6PIU?%MV30s>jp)k;t%TQqaRD;EAH+qu zm)aglh3QsRF<=XnOuQ1FaT`Cbb>X%8!32sY7e{N8-MGcy$UyXJgzn^DJzgDXFNIl< zg+1nH8x%+n>J&5{Jo2mOXqK<8OU-?S8P#z-^d3!cH{nGz(aGzFtj@TN9bcT3lP|ND zY zDjA$@gT@lmHO_%+08_V!Btl1CKd;HN^2&KwFwSMy+k|3lZuqxX-smx^^1|&tDbUBQ z-%OD}RkhOlfipXLk3S5*I%cObMnT874D?USo~sy2njuUa2quP(Zn{jE8li$c{yi*L zc}7qKPN*SGt|^9_>MLllB_#&3(N%JaxUN`9L9zg@KaZ+m*$Slu&JHFNEo*WywC$%J zy*~>6$uQJ@mD3!FD(q_NQG&OB%LBwYpeqU)idwyifY61ly9SXHFmQ3!V`q$z7 zypy>-mT6QMWJVAxIyuOCF;$<27&iI5W>_XRX+J<$|h0iz{H58@-=Rf6C@}neIe32=%8_ zG(LE+`lQ66)h?P+@!<&m657%KU6kP0mjN099)>&@45JvUU)Di=lkV)=?8A^b|A#nG5ZzX;+K>89vGzgJ0z5JbU9azQOeU>bl=u1J9N{} zs0HIfE1|t*I4_JOw{jycs#(oG@(#MbYZ?=f{c*q|!kq;H z%f&QbnN6U2r$JH3=J^eCF*u-@o;1xh!X3ADY@q9y#Fa)K93RL_;O}^YdgupRd6Zqc zMh-olA%2>;>SvwenB6{#N3wlsU!*jEvIxHVsYQ+3@&}ci+@G~X&Nmh=hF*V zI?b}xP0IeXHiI-WxF&f7U1D-e!>A}fZDY=6ccY;e;6As_K=kQvXyf@{Zq9LxyGY2V z$rQHyh?5Y*A2**m!yLHu&PtT_`nc0QU-_Naq-z700tB#jFUHZ5vmpfQT?5-}UG2f} z{rE;Kk0a$&yKlNvwYtbJ*vE!~`wE+PGpeKDrL=0OCQPWDi~0#~l0O=zF-N+MSkRK zF+CrAUPbX?0Hg3tQXTrIPo;oJ!}US5jRc-khx{dvXrk-dym69MzEN@??Pt3AOal|~ z?>8Ozq>uA7;hca_rg^lu zd`rCnL91@MPM$u)4EmHMZn*x5g*u+>Oc=1Y%F*zLezA{Uhq?oI5xotCq-e(E9w@`)!L4;o^jnoA#)4OGTXu#9pZanewxxSnj5Abx1Yz@SjgEXx|GV(?tc5`<91|rL$(IMU z@8}gz?Sy?jc=ps9vHA=p(B$LY7M^N7aS&ctcn-nIXol6J%ujAd>+}o7cSNtuMkif4 z>eK%u2VJ!z0%|W7ELSYXjK?H4Xs0wWL8QuosB@(B0^zNYXz$F8!Htidd*)|BBaqOf zm-_a6p;PBr0XkvT;h(R92L-O|CN5nKZx1E0UYUo-PM-8lV7)3pNhG^m+3c^6Bz7^K zdDD1zT<6a`1Vo`6E77OZ*_{ANYk_0~*ca??dYPy-4TrFi=zA0)dq{D4wVeq{=#fnG zFV++c5m@zc+c;dJiJ`D@X1_fV(w2=S+eA;yl!1y0x2md8eG^M#rfAh#^IYv4t}iae z%`LmldQ4n1zk3S!{IES-39KK+vZt`sX|{!>K4+uzX@gamUWn1`2r1hN6Z?whUsex) zTtzSD#ndg|^^M$m{jPgdf_dsmSfk|anH=-?Z%Rsr4mg{f#YW%~W4XV?aPOI*nDH66 z_Gz~KPcd=H96p~~u?66OH0viy@V98j8;|MZ^q!-f!Da4N_$!Zjat%l0pAXQ2``l!N zw`6Z?&*;fsOSbqkKkt~^tjt^4?95L;KE3{YG?XcXUdT2^{?*8Ji<(6~#oY20!S(5= zr7q%OaBtMv_M*ILunn4Px@NQY=gQAg+z2M~@tvlq>v$xmb}#l|jFgR%%+`@YpJPGD zn!>H|zl628Jp9<{!fWiVOA6#>E?*l5ftdbdEZ5j50co#Wq{Y?WWNP&j<|+?#*_j5x zX&be7?rwLM-|=5P1H>dWTWWU=9LFLJ)U0aqz(SEE!@`OgnUvJzoxO2kVi?Km-s+lwV%t}G_o5ZkSmnY)XrmF76Sv<-Gn&{}O#e#8SigrAl^bC05NJx;P zvB@Vt#$7^hfB|q70?Y{hK$o^IFz_}q7WW4;{;$3-Qfmt0$>4?F7uhc#*J}I=ZgGX(+QZD^sBUFHUR$VJC?$Rv6 z>a6{YcL#$fQo1QhSl6k!NKHI?bT6)F(o#F4+xAV*rzaqbeCjG?Z|ygEzI=_FdPKVf zD+ak{C_kA6BK)L#I3l6#UhW|uB)eza>o@S#%<&54v996D%Utg4%HZbBmv$QIRPyXg zW;2J&Bz|x#l_EKZND*d{S2)lnXYec>`Y%$0wf3y=%wH#Gr^O?`DdRtO5Xn2B-wwrat zR^oC#YYUAsuYu|1cT{;z#lG>52aFQht;lea9`cIM9tR4rI|=rE~oxJq5;nPlNyH%BDD zYLpkHdtiY&E;C80!}?y(s+I%avMy<4baW7B488V|&zr=#Gt<0gI$O*To;e#29&%E$ z4YS!HLdQ!M-?D{|bJK zY)MidCYk#h$PtuBrcy3P4y?W>Ecub&El8iV1JTC%(BxCat%*SMr>-BZlU~X+J6)qb zk09t8*76k0J^WnpMM#=a>~t;Ewxg1{dw9EV0BfiDfl^i7oy7CJj)L?w;1hp>jR-Xc`NY%U4{%eWLs?B$D)U6 z*3LrftWtC{jdH3~aM!(Gy70iC52$;CL<7N=ge78q@m(}dv05!I%>p+(2?@y$#h<2g zp8Hq?O6PC%JU5|52DO(|=&x@E+sBR|nvq2K6G6SecnY-+sr%K5W|z)tsJ>}HAyA^p zucWDSDabMyIiU@<2)$}&vxVf%wwBu$KP6gFABuel;_KfA9bYgY5FPhj$I?){%13rd zgBEzj^48}B&-@Dgsa9gUd|w4yA$WT4i^RL~jV2!0Z!?swdipR1Pk>Sa;dZa7PQo>T zY&$0F4z1vYIlIF+tf`uSgEUug`>ML7AF4PHr@L9mATLEMl~UBY#PoOuN*{FsPf7d} zuFdBGC-rhaDA$_yx$qlLxrV6|;yNw}nrVXb{1rW^Nv&^hw_vvGp8KZUg-i7-ke4=F zFd~js3`XpWSfB5B%plp6l^>7lo@Z7VP`xTUdr61jd5aSQjiPJ)Q?>xLSTr^J`+fi^ zV$fZo1_i-wxY(Nn0{jiTlV~?~{+uvpCXlWgLbNpg0^jaA*Hap9Zd|JugWY+1O2J9) zhOQZ|i@RmFEgAXw;JS!bG&eO0*@`JPo&VyeF=z(4c3-<(+u^sOId1 zrEg3^H3RedEZ@tF**1RCrTu@Ez`v`T-07~x5vAi+eQF0;epy~2;`H*aqN0PEp|4mx zO5$rgz`vNu&i1&Stn7RAf>SUWN*eo5al6d?t8|0?0S$m8-a-taDGxQgjB_>{q$}7z zi8K(e3ryr$pghWQ{~qXc{v)4gK=^{Y@Rf|_S7zWR_QG9=p>GK3c<|*7{dJ|LICR`a zhn~Im811Cs)dEO9WqzF%k$PgYjQ}+)nhi8}cZ*u!nirz7<%i@^1voAPGj+@$3%nWQ zdHM_lAkXHD&_N%4)Q{a?cx0_;rnyZI<9x!`Y~*lVMzGQ-pg;YoXtFYseWwznLP#m8 zE@^$^xdd+H?56#z6F6w}`VX1F$wCW_CanXMvc%q>%|E1>Py6uKxM3y)iFRxLm~_RC z=^1!L3{mocEgZ-6nNr?m=gE$H)EZm0$fVeWQw&mK?oH1*$$84Rxgv#BU0E~lw!K_T}6Lb;y;qHUaFwrFe7naI*3F{qn_^G3S?MDcg}gL@V*Xr7!xfZ z(<7N1jN=hGF3(OAtYSBiw$v=g2C28Vn4b|F6MBLC#JhUCCyTJ2G0PX@JF^&70=(T# zSZod|?}e+YQpNTjUa)Y|^iVkRB@kTKGC>fUfDB5Ni?abqm@sS_K^>}QZkGxE#=eyln4qYf8FeHEq}u} z3q^=?fgS-6$i5{njW_#fwa25!^EHL_329kBr7Oqph9NMY9}Sv;ekM5uoQ@Z8+3@uJ z``4SNJBZKQ07H|}`TNpXsD(M-=LOh9lNHA%;abF*>!rasFNmVjjR4g0hhpArF@1St z_MV=}gi`qz#p<%MeXS%Id%y-c7&-fyGCvv9;;#YaXM4##I z$8W{)R*S)I$xE+T9bBj%=jtY@;j7QHUAUc^aLt7`MA9ndOkpYJqiVc=E_QO|p#KnV z&-DZ6M}*4&aq={F=GJ&zLfQG|Qtr1hbQB_$Igd@hwM-jXuOIfIs~X7+u6q2%nqLfM z-ZH>m6$LK#ljA;!u(=N+o~*8$U(Rznw0FDLp(@R~4Z|()X{P_I`Xm7UjM|KNyx%YM zKxyr(F1*(c>sT)}g@Tet!{T3|`8af2I$ah# zNBBrYN4xVb#{=&fwJg#J&ZNLvF=xo(LR=V-!(H27(c>-|r z@fMsCZH^U(b)~R$iLmmHh}OiMLXofjCcnWqI=w&&e?kEP|AiRh{hrzn+W->e*Ws^m zx=kC@D&nr9Q4!B0@Y)vkCYJ%)8U%FA$B9Mb6|g*etPc#Rl*hN$5N>d9tc56)&Ly9& z;r(iUu;X&_>U>-OY@_+idiyNJ`e2Ih+Wtf?o@V1c@)Oxi6$H1r66_hfJ!l}>jN6S% zjD~S3UHYEpUgLJnu)EkqbejNYkmf0q#gWI(@@6NS7D%Ser`v8C6DCm^-lz63-2G;} zsh_I(5|u(_fT(lE*m$}m9A@2Uxhtu;)g(OyRf)6O(!bMNUhtbaXvU!l9CW9lLo|%Xi5Bw>-l!G4mnA+6%|@B-2{HX~AJW`8aNOK<#j_e5Z!dH9udY8Q zn3QEEj1~LKb%vx1>A@pykIP1%O`PId-JC9-?+j+7$}N*HjCziy(i{VXI8C%s&G<-E zV60G9k$hQm20k%15Y{PGI_A{nfCUQ|V0&o5Pmb&%7$54x5zT&*x4ZWoHLZ31PfQN= z%2oU!BVWF1C!kDR90k#6qqzZK^5AR>M%Kc$|74OQWclY?AlDP(-QWm3v}Ley55D4iAs(H(IRES6<}R{(F|^@;*2AQm)(Z*(D%*G z0_wBp1&6u1BEDG7&Iu$x99LDjnwoFODvxUtZQnBgWZo2@>qZ^~w}Tf%e!LH)-=HbL zw0?M*x3YgB|NeK>X*! zA9t;`!jwUXsnd8+3n>Uc-5$#6d}Ok%o(~Q<`xfI8+{d00z(lSxpN+*so&UbwiJl57 z1DU%!F6W>ZYY?IrpFX|S0lCJkB*Ta z8nnn8o;-?qnqI+iR>-#7*!Oq~K=-h}3TAzoVki6Kyp#GcKiEvwrm@(`>AtL@k0Tdz z9PKf`Nyy4tN^o3%xE5!NK?|E=UmfEp2V}jhiwN?>Z>IZ>zL>Xtuk91ewr{T^0P<9d z3X_GF?G)&T0?O^}o*y55!=-;-gLFqW$pxRt

    Mt^{Gfsy#=SZYeA*qjkT7Qx$eHJ zx}E^d092g}W+?whG809=<&t8@cG=(8#p!=AZb`r(b1fbA*XiD7e_-X>dM->9!j-Bt z{YM{C7ys?!5vBZWVG-Vo&lHjI2zM*<#!k4Nt@Lkx_5_uU{u7K~OCW}_-P>)9-Q(r0 z9J9-JC>%1fvK%vCdN9k+Hocv8IGe)d@)$pZTQJvRa~`jZMkRWJZS8=A>G7IWQfJra z1goVp=pSD}UAiUqo8C~BfO`3DdnS9g;*A+d)U_(^>Kw57^a6tphaeOLt35)M>8bcd zzQv^V--!^3n0?Gm$qO<%jW8QU9;VqNF%)6!k-G3!&T}S1q?zdsJVh{L^dzM#nH2w- zCCkB|?MbAr4-l(p z;*)0J>zZd->~MH6oex*T(XyV0;|uPeFQ>z}Egr}Z&3}Uhy;P!H6`hiU0hw&313J2I z-u3dDbd_#C2=q@AN-=)1!)yKl;KT2C$`j>>J{n(*?!hzK?j9;wF92P34JQiA)|waS z8!+UUnX~RI+1_MDgtRq_G2rWxt`R54PU^O8L7PK96QZ?|2iB8Fhv8E4ns2yb*~B1o z6uQ`p=A-9G^mq{g`<$3vK#NQuxeK%_TZKfJ=K+u!pv?cJE5jZxtx6zlpbp;>e^~UN z_=G|YEb7Te?VVq%PjGN>D$Hgd0y&xxQ9VDI!_gETDFn=e5gILjih~z64mDZ=Pz42m zQ(4%?glw-Q^03h(bp)V-UPq}-{R2(5vNbBfi5?QFw?I3JO0*KV@}pEB-N@d8qn_rC zAww{%Df1&;y4P)eL2tLYogUVo`4bE8xq=kIL@ACPx^fH1vj2G#{$bhl(!i4Z(BF z05vp3#WS&qeRB;M1-W;lHJ699!uW3CI8QAlnR^~P3;Ebjeo%6HmK3xSu~M=s{A_@? zkNc($qpimfLMinF?)6v;1rpcwu7-*c5(Bw!vh=aF$c5OiusU8Gqlc=%$F~YHlPX;s zT~1-rF#jSWduUj3~eF{1rxHY0h-&p%%Ggn>=_qiREBq~vQ!2Xzp5v$1T) zVx#xGEy*rhYh9rGDp#6_ost~U1!F@7x%3R@rc5ES9Ukvc7}HHC4{|1`g?sAR95mOO zNW3)Y=<~(_(w}Y_vPPimhmpcVxz>0Wwn-D2=XYB(P$%(YfmsGn8N2;maewh7jdnZ^ zN=t)6!Y~ssODcT)vkWFUQui(7TI(dTsLp%R(kL7?Gjz&vJ=xyLkT!&gfNpA~B+Wul zDrz^%9um(vxh5+hME#iqoxSf-ULcoC?w71$yXF2YU&E+qDF~7|L3D(D5 zecu}AFJmIVJTL;55dYh7fAzBZA6T*hdHpCpT6l=)f1#yeaKHskn1(=a1q&-{r3)cn6r2l$t z@Cn&q@Xvz0k$%|f$u?|D zN`K$YjKRBr#OgSMNVOJk?=O!d6hZy!VJ#BAAC&~^wSPG>LWIF&S_40ifp~9qqlqA<2iPAcuK`O8Ca>0o22P z{R0mw(B8;Y65&68{;w}U{$WMBRMHeLfc5vU{o~_5Pqc%iy{B`-q2Gk700TH=zq8LA78j1L-`@{RzABQ(SJMO8<7^B`Dr!z|K4^Dgx3Yd ztEd70{eb^FS@-kvzms+Ez5nZ3zt58YU$-H{Y!&+h4#~<9a0WgQ6_OGx;@9^1KfEL< AnE(I) diff --git a/docs/en/controller/design.md b/docs/en/controller/design.md index ba2de58af14..af4958a4d3e 100644 --- a/docs/en/controller/design.md +++ b/docs/en/controller/design.md @@ -112,13 +112,13 @@ According to the above, we can know the AutoSwitchHaService protocol divides log ![示意图](../image/controller/controller_design_3.png) -`current state(4byte) + Two flags(4byte) + slaveAddressLength(4byte) + slaveAddress(50byte)` +`current state(4byte) + Two flags(4byte) + slaveBrokerId(8byte)` - `Current state` represents the current HAConnectionState, which is HANDSHAKE. - Two flags are two status flags, where `isSyncFromLastFile` indicates whether to start copying from the Master's last file, and `isAsyncLearner` indicates whether the Slave is an asynchronous copy and joins the Master as a Learner. -- `slaveAddressLength` and `slaveAddress` represent the address of the Slave, which will be used later to join the SyncStateSet. +- `slaveBrokerId` represent the brokerId of the Slave, which will be used later to join the SyncStateSet. 2.AutoSwitchHaConnection (Master) will send a HandShake packet back to the Slave as follows: diff --git a/docs/en/image/controller/controller_design_3.png b/docs/en/image/controller/controller_design_3.png index 8c475bcecf1bc030cb9983c01ff07c9ed1433fb2..0379c231d46ed8ca7e6ab3a9c094b78fdfa683d8 100644 GIT binary patch literal 70160 zcmZ^~2{@GR_dh-)5<-$KMuehJLWm&>A{tj z8H~X&X3WfgdjI~v-}U+Q{(i6P@yuM0`q_q7yI)^r~ByA4(%c5i@O#tj3Lf1{O!D*0LGrr zyzG2lI9c-y(B2RR=xS-01bp0_^S8awIR)CmR{1N>Z9HhUta0jXR;*v}oUJShXSfsn z=8kT#@%)>nz`fPOciC_5pVbMAR6qOpvh;7yB~J(4ZPS`3W5E#8F(qxPSyfk|?JU%H zvA)B9c)QOl{R*usP9bg#(JBry(=syJO?Pp%7Bt;C*D{bT;erO@U87a0nMn5`m@Y{N>|etBF3d28Fn{%?a}yr@&U zT}0qZ#6_7+{VzA7&Ml=$o!l7xUnp%MWE~{FdH3^e%10UZl-`RfoX2?vXRdw7D(d6` zhv5FFo5~tj)y5^Yoi<7NhzR&hcf;gwo^w-HFO@p}w>J=ziZ8s+6cf9lPHWKo8+Te@ zF{Kius@vUk|0~V@)!GTpYGOw4Q^FnGm>?VjFkGH820BGiWxqcJ|!8h zOcZl$B+9=?CJV@MJtSRS^#1`0%6S2^;Z*<7-y;#KlWrP+(K(rogDWjIhWkz)vdkt~ zUzS(=r;*QmM?>tNowdgDn2eq?9ev%e!!|%Hkt9{fTB39HQtPGruA~n^7GTEmfH&`jxY?_B%v{3qNo9}Ir+NTYFX?DJN9u(dXVE<#yM6zk zH2}fL>P|<`G705?8b=Fwhcpbveu&}vPvJoJw9J@w2iZ5?EOc~nZ?rtA!78FHdy%gL zOAlAq(&9T2%)Xw?8vHqvuDu~uIxRb;74T_1<<0$jVMT|Jr5>=3x$D#_^#73t>(#Xj zO2&^&E|SedaP31N-<;(a+jmpse!f^C16r)LNyy55{$}tDtTk?H92~>fW*e{im=(+4tDi5a)=?A+i_i z`ohPyf8RM(A;+$9E$pd8EQ%TB%G75&%;FQPAv?voDlEeBQ#Ljx?I(X*_*TswGY7ho z3jBHBh^!Pdnu=lV!9)aMF}j4kI<jREawXoeF*;(58(0V5Px=}kyNzOFKB+~R{-MPiI1KCL0VRamTd0KX zH2k;3X0Hgia`~n`v932(iQlp>?LY7g4&mVcM#$P9d#B&*td%$z!}_Q{ywz4j+(PrF zGoIZD$-!AQ|LI<^R>eJM)@q$sd<-Clr%B)ufZ$vrlo{^#8qa^LAjBb5P1flR>zAU z5YZ4v#Knsu@;weUkJ;n^3g@~U9ZDh9Tr?GKoKL0D%-F)ZmH8t_$y-kl!2dLz$Y?^8 zDZ|eJ*C}NnX3HJo)VUsZ$06VL>BR&>)-H3O>yvzBDnOV%tyOrqb3KwloozjBQt0!u zcj=pLnFEj`G}E6g%H0`@+DW8M1}bl{leltqfXyXd-C79GPujueDNl78guzc>Q4agthDNpQwWG=pn)xJ6z0^Lk%Ez5J(bplIrp0jf&&EwN>HMQBMF^72R8`LF zLpMxWJ?Y=11=qaan!On45iYOQwkXIhuOlIaz)J~3QwitxW7E_-Y#XgPi1b7jWz}K?a}|gZ|ZYRn~ z37NG1Z|)@RCfaU)axSw6D4oRwSML`ppgS(GzSN}8wdMaaRdu?p@;wVz=}cbRFNQ25 zaO&MC{`@Nmt+rn>mwz6VwF%J$O%~L~{c~*oqcc1EIYgPaz8c(~e`!tC@o&X6V=t*{ zDJIrOe6hDI%0-1sniO801k%oE#aLE*X9ZSc1c%*v@Vd#=xvMW*3w%p1aJI5CPRY$_ zDQ8N?YM@!TV%lV58aoh@QfhCT8!KB5M0Y`GLMQSx)YlT%{ZTcsTD}G*;&uo}k|$Tq zZBE4#K#}YbtdS3?TX|rw?fs&NslGBhej0~WLZdVooX?D4{^-m9<15y#CDT;ok(Ff% zBjeAv3EkHX?w`>pGZC&amzgZ#ePMYCIza$LoRYPKi33X7dl_z|8HB}*p5?MmlQdHV zSPOHF63@!-o`}$J{d}%8+LvK*fFzYoS&1#Ro|uQCHVxMb)^AXiqueY8{yY(lp(t+!^ zVK8;EXsO?TEhvX8lR?vi>B~fGn-C%?#{4$9*TGU{EfgI-1#i$k@=aOcd2m5jXG^oL zxa&~#khxFxvO5vt5|Z4%*0Q&^jJg^wrgkt+l@YyZTfF1&0cBbZgs7BUrKgNp|{AdkQRP=B8>a9U?%j< z9uv*x%FTe@seKAUXV}u)s+!u*oiBusM&MIaORYWRK?&O}%6E+QSdv*!|o2aXz` zk>xc+85HR>FDlsS+V6(b_^P8ZWD2*zp*ObrWnop})N}y#Esi;Hd^2xmAxW(}=b#+pE>>en> zDi%uLx`YqiGDO@93blI(n)!hI>W|-5s-06ke9aWRgChqJ(ep3t8Bmz^^}o*Y4_H6P zQ|H^;9&NtFvlx?J3JzJu^NiYxT)Kf%Stv<=nlea#suX5FtUb&BmUA*v-uHV&2IC1^ z`P)_JZu{l9ch`9TX2`Jj`l(O3fHL&dN1yOB0d{@HWE=3t+oag46`DFw) ze`xdKz{J=%*>u=64%s}|`sq4XF4wzLLBCnW>9tSVH#{WEakVq(ztJ$yg+5r_RQBK8 z={Tv!b-BYl%l`LdUs|M=nX2X>?;DV^89pMPJv4A50?eA7!|R7Z5@~`20d+X~Zh0k8 zHII_ij!L`rqnN>j`xm`o!4LP zbDw|oa%F|^caN`=Jk;fY)7_u%ofqu7MBygUju%-@Chfis@#Ji0W1lQn>p1d#*I_n* zMN02MUmUZDZ0Uj!E3dwIIIs1xKsoygaJHthyx99!T3>JrNshN+j42a4mINA=C`hjj z32IH#Z_P*^QETYHW)L*`964h6WXtc7D?H(h{=Wz7RtCZy#Mf&E8VtB%V)6HGj26XK zv-swI*%mj)5uo4HBJ@SfE%2y;BL+?K0&rYLswrUgXUk;d`%ED+5llrXRw z9%!$KF{<|k2_@z*c>&(qsM@bw3d1i*IJ-S#uA0~Ab3bx__gHD18#+!*aXAC*`-se= z{(N2oHa2T%_s(u`S%USQzz#Z~atolwqd4{cb`7x_As0DtSYa*uVr-tq7cJ6U>x6EqyN_l(~6C zVgNi18?8IQ!RO(=wix2q#p8^x!#ZFSef!;Cu_JP8GDB*k8JhT1|0kf}3B+xmW!u$i zy+=%}Q8eu%+@$A`D6*`e-=UkDp9tEIdTo59wP z``m+03_Ux+DARPvB3(-&k3rxd=$EV1+hK<`ume4I zwLj6a)0GhyclR*#A_o_QbX52n8+6!DGfL=UNzu@jv<{nO9&3=-Y@Pz8C9$OKw%@=@ zO70B@uH-?Xd2^c=*Ds2d865Y8xF6P>uo^c#$D=SladaG>r?w#nX;!&jHmEX4F3+h6 zA}0Fh9SlC@qINbVH<~qNd{ndj(O)ZPd_WTxcB}vh_Uz96>4OzSwhFYazC~I=tZeX& zot$Tr=y!~VF^CN>uVUV0mgZmlxvtgySz_Gl2M-3*pd*6MuQZP`-SFP&^X(=+leelG zFZ-$T4f**NP`q#-I2!!xS1ikiw+pUdDvX4JjGMeyRFuC*JbtcQ^eiI!fwgy4*P1`k z&4%D$?bP`_7-oXsk?Kh9lRIQ2xyK7w&EC^$-zw`PhF2@V);2Z~CMq7?^(mLOUUaVobLAtZatI z4~Q3@Xw%Ct-msO`m)&qE@w;Vjv@!K=te8vQ;AAT;4wk)VJC~iNVT-S{&DtLnRB56% z%kznL?E~d)j>LEdC>vHuH|0kg=dwFmLanSG4+23Q?hh=3;&)?Nzv^xLJnZV^fp_-~ zs$9S6Gad8HXSxHq0+!*1_hvrt-RgVlUW`HpTEDns?_;ca5M^o`(9}{uB(l_NvwK?l zkL02*D7sW`RZS6zGBB^FUH3TnWIx;6m&C6=h{aT4L{vzFZ|tl(TTcbE)^V2r^nB4u1uMFEL)?i8dRX(8w$ z8tq8fpX`N{8C~knfimv9M?Px42#~nVky54^@@UjBW&zK%XiOTN7(bUam9F5m(~3*Z zQk$Huea6O>1aIt+eY&3zIFpL}SoIuV?SK21I3>l9YLAUFmE}x+e<8+z<}=m53BKt= z!Rs~?`5l+tZ3H&r8@A=Zu3WH#25yoR*(-GsX3(ci&4 zPME^4dF%N=XbHhxi1TyuJL~e8kBKE;6`OpXl71M5E|DHm(iHG8wC#fzPwwQ&6gM>8 zr0&OxHr8%Fp-j8Lh|z@Qn=_z>V7EZKwqLp<^ThVS3^b-&wv=LpgzQbH<>h_&@uL(k zrM`?KWbkA5424igCvtxh)MxhNT|cty9?b)pADXjj+quW|dFgD@_EgZ*u7YN_R1^*l zT2HyI(}^1WRRP<}SdYvbr9{j_i9Ao8+CA!?Z11?j%@<3!xNcqeI|M#@U8K+A688JD zu9JO=HIIhJ)tYG5U&*B&OB^&*hfd#-t!STK20)Y${}QN>b;j1>;YR0`JZ(<~Spi{H zr7rYHo6ca(m-@U1BxR+OWV{TysHoC&Rbo5)u!z*XLLc5gUOarZ+9rxwH*~mdveEA- z#%=Y|K6)+(Uhg#(2O_(oR^@XG`k(&*zwvJ%%x-)O*L+7hV}#4>7~{@oswr)xv$(_355Vg&3E zVdzu_bLj-aLkK)DMm{t0iBDO*VB~7ljPsngi^&UjO^<#nL-PL5Y>z?m1P@y(sYV9o zLXh`);%VNtF$<$CtK^?t^Dhf>7{3Ou!NBP%p1lg`UYhYI{KOuU1s^Hqo>+Tu0eU2X zQuW`r9r)odan+&#E$+4z_uZ$En8Xh|tc9Ihz!fWBmg!+?19*n)3-qc{$9O1fn4@}a zEMTxcz)m9Ep)$J*^TNg(QndX&4A7a-8Q8JED^df{ zH~zr@(NeL3N0tSwb`Mf^4(BS!wK&r*b0C=iJh`???G*ulfKm=^Lx?yad-|=rS-^}8 z0e%d6hERC0W^#@PFL>idjz3s7DYMF}YX-52`sTd7b*Msq*_2FkJ$CP%6gnF78N%sm z0dfn9Z+~BM_#Syu&%ZomtFG0L z@M~@e>ZZB&pbf`HVlkr3W^V$tIYZbc!fva04%jzRs>ZK@8eH`3u@v}m67|6rh=ZXj zKX9)^#yyaXm7)-qU7*MBO{uI#KAPkV9?C6WT)%PZ9Dizt^P9MfA=7Koc)Wn(jB&<- z-(T)}k7BFYBkvaFOJCv4#kW#kUON?bQqETU+*RFU>78`-pkH@|?aAr%757|OMPy?% z8s^?8|HwOS*Gu=w99bEcQF_XhWuY9`CRx43ldZ(2P+$#{s<-ZuXy>MF9VwlqJjvj} zH5thOwA3MTKNqqEJxttPJXWBjS?-B8&nGK2x|K+N!~rK8velQ?o)j)IB{M4oPHBR( z>Ig&~@VV1$c&^;cs9~t7 zuWz6(?J}U*Q|z9$@rZsBXe)x7DH)<)YvQy<@WnGM(Ef+aw0dx z&}1|bP7Shm#eVj2IpaCQTM7eooal0tu0LG(l6EVsDZF*m2`>C8zdjImA}KLL3AN%E z@r(8YoFi+k9j~~wKjIC;ZM-Sub<_>3; z-8#P-18h6!ZY@ZKGc6Qox1>))-t??{H?E>^>USQaSh|w9vq5u$GgPtTiVbAe-k4U- zmU&9b=!BU>p$lvJt*HZ1k*1+aoX7FUL~K~V@$B6m!wp4QF$debx_0y7?uFn01)kQl zslD8##BEF6zXileRQrByc_(i9MIrxd^lp=?V zYT;qD(?$hJ&Yjo1#~;OcyG4BRW-4^Mgo_8oyAe-cDZ|mcOwnddZEX0KS6+N)j6FiJ z`PjVT`Ccwcr=R!BqT(#)#5^vY@a-FmMAG|@svct?Ksf@DUS~V?ZZPD!T6JxMG&ABL zP)v<;@>l(L>|Zh7dmzy=J>j=|5tJ>k@*sTcY}QnkPrh*0<6~(YepsP-|5cfIj+V7@ zRn9~bVEpDASFtw3W049Qac zLkj-zctk?G|D*OnhO}+RzLKkOIa*^%xJT6gh!wKL!mQsAg$+40|m-G2DFx3s@t~kK~KlPg57_a$BgR5W`zXg;6%g9Y+ z$U$q!8N0(7J#p$G_QZ;Z&QFgwG__M!!IxEG3i*(Ghi+7l=_FSLyHS|R3RjE;R=5?z z4){`1O7pe$2*;_r0z^S%=t*5#yZR(6HM0sgN9Q0u2{jS^z{~yWWNI92 z@uk4P1*JDt8*_HX&eIf18HSEu(S_3rQYOorpc@Z)RI4)swSNq@A2m0dqTzSCvWZ#p z@Xh4MB;rE}C`QwMX2NhN-59`_2tUiYAa-UPrhn%V?K0KX)$~nKM^z))@4(2==T1`Zs)o8d`Ux?Yd%*#q&srudFi>Z`2 z=XH$yUF*eUArVG$r4NtwE#G|_0*7Ab^I=Lzzjeu{ScYTnM^apDBW@3DMf{!mK(LR7 z*e+>}Z!h`;LdX=Cu7m9+@}frf;s31#5I~tzI!^X}oj^?22KJ9S)fN*5fhrc3Ph{PD z64dn*Y~tGizYny~re3cJq9P(=!-CwfyU#D4piBm&&V(Kj>E2Ashz_q!?7bBZGbZk{ z6NMSiJ_{9UdCjn%u{jb}^5-E(U>*>Memk+_lY)1ZW)U6O%+eky|R_R~|HSDRRR>B2?O6R5-20V?2nc!DGsv68 zm-R8^cJRaqiZHv=Eyw8cU`y_4S`{oYLyeTR1`8H(^VjsPR^$-=yjoD;5L`oWSd4PE zIiUjOV$dQUdo22E2TX-^7o3iWm5gUezomSA)$k-HYZE9{jH#_>Xqk3#?ew}wl=33nwe=o49<#UJ7talk`0f2mClmay5KDMSS zl~+Ad;%pR4PS?a1?u+M_MTF!P1`a_UbIT`2U}I}@YfE!uYp}H?*xLGq0=v40N44Y! zb%tF=`zAW%_V~i28sZD}@yOo)5Yb!JUkFwSm<%q}eHdB|Nd%1EReP?p?)pggxO|7% zF;tvCJSVACl|nzIr#-1Qe&MR#iV6Gp44X)@sh?RDY`nO&e8-Z-c9`;BtX4}apr(a< zf1p5KNnoPYDdFP{A6zJV*QLU5IBTK9H+|hS{x2{QT5IqS5WdQxvG!x%w7lc@bAV9B zBwFxui@M*v=u?a`u;tK<7X z^-@U0rI+$<3|}7$*;>GSW|pjsA5wjs0YC#x+5`m>sk}=@py^T=p znltu?4^mIymLrawga=+1w*WD$`w0z0)6!x-s1Z67t}m6{<6<4+1F8ERSGdjWmh{BY z^?*VbC)iOZRY{+W`52ErfBtO0RQk2?Aqj$lo?4(8+6}Jj>1e}o$~AD%Z8foLE+dUJ zIH|>F1tWxj>@Es8!*JWH(_o+_d2wyxz7b4#mx>WVgL2C}pEk;Xz*RROIrq#}52x zXm~zx&98pekD71m###pV!bQgZWCz5p9~S|1<(lb zxLs}1F?6LWwl@bIez;wbxFp7)T&mE6z0Zv}+soPs3A#PsUl$Ca9m^MRzz zA)T3JK`?pa`QH zTZ7Dw$@daMf5(C+Ow3kSHiW_&7s|$RcnS9E`C~GOdn=(@pbXh zq2WO;(slE%K?>K8Y7k_99|z6(@XaO5MsJ03C@Bf)71*@oJZOxg4T5&{O!z32r#BJA zot5i6PK4nmr2WQ0Rvv{OWZ4(J)r*J3T)LYIH-F`0qRU2Z(+Q3z4FXU6_>D8x?r&1) z0SfZP92}jPzYKFU`PPLI)4V3>=sZpQp+^IY4A3(Wnt~P_2>iy~EqdoF*#kRV34YJJ zuM3Bb%7-rz{JfwQNr>$N)-cgrdg0f42ASm8Mp_)y@?~8I2^!|vVWgkG40_vA^`z`8 z6UH=WVMYzn>VmPpb^9;OMf+%6G0iH5Ygi!GuG7wGTy0;^i+3SrD(()=XwsPpq@8m9 zB_ZEiP&j4bG-4H@5<7J>=6$=F^OuXjN&Ypu-{t#Tmf=RAN^@jhKDpp(^GZFHxLfZK zy}S2dU-0(s4d|W6%0Ek`WyE`dD#2~jJn(ty{PdB(>IZ>KFL$r;0Zzlgl=@{3==s-c29j$4qHNZ|5gf z6-!A$3lX1(@UBO$H=;CHwaGJ>*UHK*1_lO&hih-_W{E43bg@ zt6i>}t`(?-6&ePIa1B2RkE%`fW%WYG1RIq^GOo z&x$uqNi71>NBpv*nrlgdtV6Ta-H**1=T&2HNTv?6zm@sfD27x6#{izQu_@I zWObQ5kBC%9bYkiT_yA|>F|54SAy#r)}G65t!=b@$XSz^qqw2FJ`-o-y>O)FGN-2Rl%a zI6VWFB47zPnmE2Q^AO}Ze&ZWuOHQi`54K4gbPo!loI(t4R80j1be-p@d|tnlISmm$ zQJvycgN;pK_%%E`t3kFJ2}o6GX(+M4VW_KjYJjO9??*j+XkuaZ9+k4Sm+MLOAbVkG z(KH*WZ0#(R2FVGlRn}Kl7Uc-BD)fKscS{TC`JDNVcwn$OCsS!i&6_8mB^~H(&w;8@ zVRp$bvZBFDXuVGOFa!`b-Q(?Y%*FG>P)jLu{K8aoS$%!rY*VMlB9m>a5m%uHnki49 zV0Xzl_{2|@w9s^78`GHKll%N!yJxl{p4qfedUs`BUgk*sRX|~fphhXjpImU#Q^>m9 zC6=+N_rT2eJzCtIC%NyQ1ilDKH!x#U3i-p;Ep|3$Gs*Ahzv9>rkWbcUy^jYNS{vA8 z(uV2pgsS9L2i>f@txrpkjCW0YwLnA5GRdZmrP>%91Q8_kivH2v$|;0tAB zW5$>1wjAI}f^S^|ecwu##VP}6&bE(3Xjv6>@@51s0^fQtjMp5ub2A2Ut%BEH$J}ZN zwwlkmP)K8UxPxV*NK;E?)%jbP==7}!A@>vL^FC*aV2{9b4=&;%iS;UKEzi5J0Qk|x zgF;HSq`tBe%JEFDO6?cX!1SfeYCG=h?}mrB4rtm0$Zh%^lDSke0}3@FX~Y0G;zjP= zeHgw)3y{U11Pj*ruE3$D=GI%K)bS6i-{lG$+(-~i#P<@aUu=@zk;@sZR4?*(QhZWE z9Q~Wjs>Xt^SCv9-r^DI=iL|)xhC0){m0L!c;177UCwExnx0V4yG+3@zAm)!{JdtTG z5Ov*o{JR29ZUfP^Aq({;EUnr6p|!@ptGjg0108K(JjR}!v(%tPc1#e0j~0S0DM!B4 zka72Cem}rqV`Ceyp>lZLdlT8$TXWl`RI-&~6X_V6mOk?l7=y#&5j9;C4c%(rmj*r} zM{fqMj2Moankg?qo_R>$PR87V-9bFuWTyEw2VhN^QMTHD7s`}{>i?4Z{6e|$D*Z-6 zP(#$mhea>DCF4Hvk=YmhUk$Q;(Rgp8O~3Bp9%f-+{cKRULLT1Xo^mp5Kg@t_MMt}E z+p4C^%MmvHxG?ll{JGfho_C8pEmXV>UxfX;N3+{8@8Z{FR)sviAOvrT%T@X-tFuBE zViC1B6~yt~F$Oxjgxtsc0(vYmt8itFNY05qA?3h~5v*K50Ih7M>%d*q_P8v5rh{D^ z16jWet&#eW{(LKoU5FM-#fCP*iMyd2?e~}#lOOYAB;(@w1<*po1#AS~0jf@}u$j*t zzkL!TAzeKoKLPXZ&4f{BsHEvPu_I?khGXk)QomOo`iJ#T)y{Iq=vACQcm33BE5Qi0 zR`g8VNBm$|QSTTYB6xa_<}2r9=8ZiD+&-A$5ea=rODQOQTWW?Kug{$L;Vgc*0mCg1 z`E(`SR2(;W64Fj``_N9TVGqkDZ}Wsw)@jKwh{>m~c@;bRu`lHcY0)VSl}XmAh12DV zUzq%A*Uw@Iz8%;nxFPsX$>}GdsI*x7TG*H=>LKYF54E+Diw0k=84A%rXV|1$>!%IH zc@6zNHFA2piUQ5~r_7~pZ>YR*bbsWkXKZY0IrJ*IYnbs&JvG?SZ{z^D7-0cK{lN!! z%H-0r2X0q5wGWsg3img7V)WcSJgzBc246qCb;`KB(eh{tgL>7yw0gDfJ;>{cv&oYu zrh&TKEu|NiBoIN{wRc^1&yx0IPj`$>m~v;U`Qk>ew>#-SesKbz0`-q{`93Q~%chx` zKDvF$=f1E)c^M=F{epIyeF?H7-^Vo%(;(uR4~i7rFgucl8|vSA+@K=zxu#eWbU1n3 z2vXe5m;#jG=Nrviva4{`a}++bF|Wk6#|ot6Lw;$8q2{FHJ#Ezd!77T~V{X*#*YEYZ zj;QJ4xX+(ysnoT3zr#{1>_a@rZr+#2P!NW#=Gi;rSeV3X9JDYyB}#IW?3K9IaM5k6 zM`4W#|1fE5kX?8TX?uwHjjJ}3n(p$S7i7OMUwO1lIiz`IBqA#xU+Tx<;aL;QM<3zA zS3Q3J3Pv9KkjTaTM{*bbSXoog>Nxp-C%ZhY#4xd2!yn&eRc<+RGg!I_zxY4J)8Kv< zvDXQBH=YIRlN3fCuxGq5TEd=c_f3t#%O4*9}#`;;4{M84m& zV#iK@Cl1_y{n{Wca3o94vqck;cTv=#gD}GMReDvZIgMpq$%y7&)jLm>C7++m%WM{< z^c^Xd_JyAHbl5CC>9G`@L9QxzB#7hM*It^!G1)H)kyrLG6iQ!PnLyI!>}St&Vz$30 zyMH-2(ib7qq>VwcbR(XCi+DNeKy(lrZxf{x1VmM}clfPH6y(d)k*eAY2fY0Bozu{U z9!u?EMIUZ4V!%|mFPCxhc~;pk#ib?^3X}Y4C;Iv<4vwfXJH!Ksjn9jTn%t%o)_uO) z-t*S%VtIMZ)NOBXf>6@oXv<;$2j?g{`=l;KzwwurwR2nX2DT;Sgv{FY<5*Fp%=!N5 zoikyC(fNoj!&l0hz9!UxgAp}~4gxk`2%M}6I-(_&rQ%o$Q2{Dk$Cb~MC`1{I4RM4< z-2=z1M_bYA*O|K`x(qh5w~$$ete;h-UrV={G|)_L z>yC%mE)vmbT20E&6@HcAh5QiS?MgYGO$&-EEuF@g1m9%QT3Zw9Re4_)^lc$-UB!NM zGZQN$D_Lc5%ys_iOv|w;FE0qIY}80`Gc>I(16mlHwGq?(*Ia_xcpDD#T^<0{-uM&3 z&|ptU9t~Q;=gxZw@S|y$4pK^vmC-oVx|s-x(Y#T3dw&Uq+)G7S-}0-Ym1hvkW>Q8G z;OS4C44R#w=R>ptk7_FvfwUp26c~LN&H;sfe z4^ryVai8p1ta+JnBuIK;2&{_S?gR-=*5I@b9;i@giD8W;2__8FQy&Uz7Yzz0r|#FQ z{yVJ51M&1pMqDtw$S!~3m(`eOjkBd-eO69QatG3$ir?yjR)&}x$1C~&n4DT-q}S>z z;71f=Crw2kC6+MW0i@YhRFAnb1_7A9%5mDB6>$An=jOfC|>)t=mI+9s|9It`rgSQ}}g@fc}b84gBqGJCznuZw1eLYnxA+9uq zN!s2eK6CwopaEI;p12}TA+VAY$PtK#>s1+P*Jq_k2HPtv9Zz}`r`$`6JWCOW#1l41 z>Laz`|{?l#p&+SkawKaPu$ME76X5}-C`$>5^7@!hga0J16*-FaLBB^}zt74e`8E9^q zJgX>|yM?y31k=6QL#V!RoPqvpU46z$k@g~T#P>$3-*D1Skb|hGu1wpcY|s_dAbD(N1$kCHaj)4B$~o`35< z+oT&$>v;j{+d9k7ir}SQTH-nMw}$-ny*Zp8ab008_}#!=@~m>6ZT!LELCJkRmcdm| zLRVG)HG^ax)TIBUj`CEBH+ zxW-g=duMSBdBU%K`LAWAA_&k>8?F_}x;s;ZcG?5DwI@@j3@eldj$ zlMm5H5LNt0ZetDox{70mM9Z#+d1U5(@Uwsp8EEb97)z!1_np61^8ss?8KZpP6G3*b zGJd5l99-3GmBjH&{3%XlLlecQ-`69-!TJRe{jgue0w7mqdP~DFss!tLGDe4eV@$}o z`oSwZ()u{1Zi?kI*E_4V0)IhnQK|5pOL&)@26OMt)HsJL~c@z=tMk76o^D)+gNdLb$AN4+^!y+*)p zpO|d?uGqE`Tj;aimoD&M*C$`GcilQ|nIktUe5QgaYHk#~+scyydCYyz1{Y6#zF$@q zxsNJYGOXX`VpW0Bnx>$KyL23<3YK=6?Oy!70L4}`kG%{3<+dZ?E;V|Uvu_09(-5+4 za~#UI#r5Qww5uJvrV8>OMH$t#T;F%OSMxgq+%LcG{=w_`xED&RR6j1q|I__)i|CFA z8@DY4{WU>Ht=h$&MYbTeiQYa%15%_IPM)kIAU;?FCRwf0XXQ~4b83Y za+X%|pVD_{AG%qzU1v;v0qr&ar#k@n3ZW1QIi~d}Jjb)CFmwKkt!WTpf>Hh(>(U^z ziTw&SxBFsw!Vr(uH$2*`G}oV9L|nF&wfgr5Iu5yTK3jWV#S2V&X3R0_Mxr;}jtinf z-CxGyCs}26LW4_aRtJSwRtAln6{vUNM^?yX=OTPQJtms7w>M+ng;(cYUKhs^!Tl&ibAwG`H&Y(#8P8nwwMMNAna=JQscrrER~Am8C0V@4SWa<7=y9z z(E08)9JKHMwHFBx3t0`C4+YH0-;fv=d!!?BTKC65T_^T5obIdH3p&@77@_Ay2Bj_7 zFaSp;tGv3y9IcN1d_JI5+r8t$a19^^^%jYqLVW~1ckJsss-Xp@>5U*Mw>cG+RAi0Y zSf=)&JL0OeJ9fg%ARXKh2z1jok9De1_u zVJ2t1Ql_J&BbI4O#>~&<1+4?c#W#dmyfSo0m%I42%5{Lfz5N@yaVv|&J`48#Q{$4{ zZWY=9uInW~IyYnPhZMA{pj4_sZln83v?y|lGOkFe3_X$}#sB~UQNf#=C-p!iv7{~+ zl5kdD$)b!JEQpkexRieg4;w&))YIm_44=T7T? zmDIj=*6z!gO@&Xmfr$N=ZixI%QCWhmR@7CX{-rK9zO!X;Y64lP;X{5uUd z0%=+5LA#)bGizFK!`bq^E}@IkF#RWo2x|7CKrvtgX+DA|x;)qD9O!P}6ZuNh7gH~Z{iWM~E2Rip{p-tjSVrdih8{mbAhT6m0SxJ;ED7;_VW3dzVS6&U&As!}qqCPMpPmwa>A8i>zgMefk*Efr3vG*nX1An4}^FZUlYbIJQ81!QC+e5k4YYm4ihtNY8 z=~|WGx#EW6xK9XA*X8m=xamyYSK{+}QCj)=tZS;qt?nQ zQ$pIpnfIRmI?{3QgY`4=zR>L(@(xCAMN2M8f@lgJ~7p)}e~^?<0Wv?B5XcHmSN1{rSw_{%nHb zY@qEPzqo8p&1ZXR!I&@&47UnFwM$aUAQXpoP`08~rl?4BWq0!f;1uk(+r&T$Vs3wD z;@;rn-wB|4Y*^j$x43u5?|fX0hvgbP$cM7<*lzUn0xxXxh{q>s+~mWdfP$p5+S^v3 zHI=PH>_Py<13Dl3c4YM%&Twni3$zw4O?L$H=bUKHxeJ_!i5(M+h|| z2WQ6c^yie0x5i9gw}}Y&nG15^tmej{$U+sQd=MYuGYFYo;*-QBE+jwj1GivQ(QX`7Ww9 z*uCOEEN*z>0JIKP0J@g!u>C+Tv5%P8HXk9@;RQZ z&!gI?Qu}hqws}dsER@<~scj^!|A(bR;yhf**ozo|2-nP_w3G?({-~RZk8*qZ^BYXagt)Y>c$W|k%dG<)C&u$yHflrh!9`@Ivk%?$MheWSH7nITsUUK1 zG?egOjxzcFeOkwI@w?COSORrJnO@8gvASx!jw@`vf0-!D7P{b)>R$TIzH#YC)Y3A? z!WE8=7QK=zVM+=~%au_J7!2qj&U{MKAZpHyw)vG`c&M8b?P6>Hh9WCl5%;PLt}j(c z5I^O@4ivsatMj71hTU*84_xR>%Nxy8Bi{+R6R1HcY@v|bmL%RT*xMVv_a86wXM-zq z{r+fxEhb1vH-yI?LTqM!fa-izColMKiO}D+0c+*)Xf1mOhCF;b|0PBTalhecjHmVX z-Uk(J{mi?JX$HJELwfout9ZWMloy)(T5fPmJLba)vpCVYdP;DH3@sKS(f|~$9MsMb z(3dLqnRzeu#`A%KHxw(dRcdB)FOW-xRs_{6d)o%yD@Z3I`n`;Zm|1$b0tq^jVQAQR z3n_49HNpva9yX~6>FZ;B7CPR3_9>j{K>4rhaKKf>$%n3oVC{m6fA`P}UaIP)S@Y)1 z9JRzA=@~kUEA<96r@LqExAIxph*)OE7fK8q(fZa*p*7iQd<+q>X{Q`RbzcsYZ|6JU zlL^Ak#gbCa!U{?&C&(XGSN*eLrxODG)zqa^|3ALoJD%$I{~v#qmLy3?;$&y9jI5KD zWF=V{CxmP=j(sYGBuRG2i0pB)IkNZO^VpkXABQu3Po>xUt@r2q;~zISp4aobuE)6F z?~liIoizZf1N*&0D_j-~_S{nb$Na;p_LS-S8}GA{Ia0+n z@PkG>k(jQ1lI@o?^8prcps5y_$RvHc9X!a2zbC_3{XP5kO2;m8#uLct{kiO%Mz>hm zF&sO{HjEN!t!SRJWO305K7U;AX-sTGTfAL~eg|GDadYr<)qo#c8%Fx5K4;Ako@~U=oD&GV87pT(sUiYW@=)$=z zo8Rq!EsJ@0F6nfB%9cTmNUR}qpZ(vrP~qqfaX8Y6-y4z zto>kMW!L4Pn` zyCG~*wuQDcAloRpTa72tY9*4LKGL!5Z}B7+l;q`PWaJcOATkOPQ_FxK7FR8zUVKlt z?1d#`^I09@jvh($Op8s=ckZt+dK*a^`@QzDYfFID!JLftwWQUkvb3!r95j45K>hVE?9quIu5;ZzwGl@U{gnomh$ghg;c(+ z7~-mpk##nMc7(rKy2olu z@85mcIjocd`efF&4@seCuD3C44h{~!xk4ErEs{f^pMx(G03;fZ3fux-0_X$xs6b?P z!3bavG~G?jq6K0+x_oPb@!rtsso*Q}(#`j#WF3rujpyXvh&7thD7yn$AWy`4p%$y$Fh zkd){C##AlH05Q*z%na2mk_s5kacy&QK6R;jJWEHZ7UrA)^p2Z7Hn8Gvcn}rfY{u9h zsZp<^(6Y!a_q+9^|+GU%+@qQ1sqpQn9sF~mn^(x zvYSi5!7jcF*qI8uqOPE~Voz{07pL%RY|z{0ZDnUp$`qnx-trL>bJi?U@HSa8v-0NV z<74Cr4FG1H5Q%WVzucoJxWK9$bCaAd^X!=ePuM}3Sm13gj*$)&VR0)Ti`bnL;keJX zj`CJ&@!XDIIz?() zol4YoDsuTOjG?zU@Svc%$a;O}%vgdo_~*YXL}|K+E^241;k3L-Hkx8xa(n237o;Y0g5_J*BQ-C zioyGGa!;S=OHepVc}a}fA)pH&d_Q5x2|d5O2C>Gi5IFg6yUHcZRs-jK6AIlh=o~}l zPsF_%S(vm=y{lk{t#;t_k%tFNRF{T|MiT1wRtWa&Q6Asjn?#0~nXm?m?pj)H!!4Wz z4674h+!$QEeF%>6WHheXI@H1~?z^&D+Ie}jqI#UqW?5)keQ^3AW%0dA%4+G|_aRX$ z{MV0Hj9!YK&&B{JY){`=_w4D}Ou?`q6FLSOWv<2g&Y`(^sw@$uJ$t!p=Z{S1c!duiKUu{@LaO*~Ei&P4takK^<%)V2Z-PRYT8)g3H!=fk zx+i07Td})$WHTwBSwMHj5L)oC@>iQ&fwk!vPwSx`z z_M3XazFcmcIfPbjt)ze}Fhmq8(LQtNY>XwZ0V5S`-22rZuk$B^UBk>%w&`AKJ558O z+cdQD`a$?r%DV52^Hz+;IX6+(ZrRI0>65kJBKS<@(F zy64ciFVmxi=DdSXeC6+V=tiuE!$vCnZdi$QJV71knJd?n45bir3R^SG3nk#o)~$EP zTdDD_c8D($9^R`Zn;#Ni=cHw;K$qD%9$>bRI0Y+w(c8BZ>p69b8cEaBYgl{;Yl7}t zi=y0W$SB!=EO;tJ0BrBdo&T97_qsCA!vNBES!bgH|4G_k&p$e~?Eb24WXZN;gYzoNP=xBRNnH!uv$HMTd7#2tcEwK#5hW9!J^1G)Iw zLAeP`EtARoH**7VT_M{RkFdw{@6t1|r(R3b!nQ@u<5ytT70#83c=rb{p5qg6Z~X#( z@Dc#+jAPhsesUd@Yv66aR&Wq4Bg-_Omdn_QUhp2DTQHz&X5VHsnW^;Dt0W}1h=lmm zZygR`n;qx<<>iFOEBnXscQ}Nn*0`$RkvRdXHwSlrI7`nzbr!o|h4-CJcZ2OkhC>MG zSo04gyAmW8YI;f&UjeV4xIzUb-Cn7t z@fX`@*bj`dOpHTcS(VL+G}Yp%qR*n+zW3d{&y+dY8l*RGkle)909YQ5?4^k1P5b<_<>!FK#VPM`jD}?A z$UIl;d%DQyz{>aSGe5s%CU7{jNAwHV{7z2EZG-w8u~3=k>zT&st?Osv!!$2kHNNWN z=YaDk#CU3rbCcJnZ@vf))P$($Fd1DH1`D*Mlo=U&Gs^i!-!baZ98|r-n4D=ot9`z= zTEvU>>F#DEqkH>0va%b?=zcge->PZ)jj#q@kvj|9ta|JFwlWZ%=02!>Ic+9+4RYq( zxlsZRBrlbzu3XiQiUeX*3d_p2xSk%57QK5iChkKu8!0*yid*-@y#Cph_XbSJ6xqZF zJm?o8Fvk?RH%5AI;zJmbLMcmWC%uLPS*guR)9V*F#8Ti9AGQ0$|J9SopW_)(bL~`P z&<#kxIfFVpKygjp_`|#F+LJ8fZ47cXj2SdW8vO1KVqq9P?Zbr(*HpVZ=a4(prr8OI zXU&Ba;iS8RNPYZ(Kbo*@CpAhy6FTpqw#Rr0{%2`2k#&c?2OHjNp*|GISd6!AepLk_ zxt`j&PapzWyb9>@yfsY8nN|2SJYvC)2WC@WT}43RclOtF8BM08FkrmFnC~nwIem$j zd4*H>cNNo{z2*n4hhQIE5bU>xYOS=*cw1d%F*9r`&0Gh}{BA98c0%6NGEKB?DRHAwJj#&D zgBh@gERqbrf9U0bunKF{r#$RGX_2ji@^4i@S{>gxjVz+l(n<7v$vDZziq3 ze>#w=-)IzVlCc2^>3mL8uN5Z8c2(F@tCEkrpGQzIv!-yi{9Tpnx2X9;=B~W@jj&wJs zVn~KHWTnR`JqR|0^FU1SQ2ES^pNHz(#Vdl*l7b9IW&sHv#OJ0lxbs@DZ_+b723~sv3d;*61#Cc2>%k7-E-VJYLxB2;6ars(1Uu3 z_M*^R@k%d;^MWKm*JepZIsH_X6vVlji!U}{iFRDXWlgrGEhCc6*xOPr;K7}G;&MLz z%Y{C`(^t^Ho*sfIWvMTFgL6JS^_zPg#VD-$w3Qh}T~0=-O&^6xSxM?Ug@RegbEV8nl5CU=7begO!s$XA_QzH`)4p0FLiD2WJ2 z2o6cBMA7lwQm0#}Gs^Z);oMnYzenWWb>2lasQ^&=e(|Ih7} zyq<$pRB_BYaAkM?Yy$Yos)S4L%*-k@sZ~*%4GPkHn6=? z>{DfzGu|1}shfez%QeuO$XW7UOIA1!n4I=@|20B8FyHHYf&KezLHRN0@{^P$4dF8Vmr2ZYq0(^48TB#Esj71j z^8uZmuN&2M(W~~CcF=uscip!Att7mK+1e+YFKxBXg=%mNRaH&R8Zn)H@EwWzvuvyX;;GXjjnI+kgaBy;v^k`uG(Q z;A*aO$&aloOnhwfL45HFS?(hRlHddt{xDIl$6N-2tZ@%C5HD)v7;9uQA{!f|5HieM zTsJaLIr{JR$})5KFQ!NZ0a1xyDn(9l_omogr>Zt!4~I4tpMUiS4R$A6-RV*a)@I@2 z;p4x{ck30_{$!k)hX-8$s`J#o(D>E*`In;zTY5)*-pQ(SFl4ECRc>iR4Rc500-YB( z)PsKF`U@CWNA=f$l00HT##HW0!LbisODkK3P(oQ5g5~9ys2)2;$x#QlCt1r~p>hFb zFIXD9m(RG($-z~vS!+b1?Nmnsd@CT-?(u@ z#8%(Z($c}f;pxbyyp0R73%`THhRrmx3Raag{Qv;0A>fIi4qzpp!4MGOgWO4~l{MWXPw=w=GGKz8Qa&zE|BtbtHE2x~VT z*Qj=2=YFjws?N^FM%?S5FTT>lwtt-Q(_P2;`T4nQy>eEEv65G>{yF@!3P*E)p6>hpn$i}R7#;*;-7ar;An#lUpPCZan?dWL6yEZlk z5)+>~_4?lptP&zjRC;_nPo|}%C45KNGAsKjPTfIW{pr)EPo8{+90lt?DHL|;0MQYS z`6L3Zn`(C}z66Xygu_wu8i?6HLzco9%00^BWIN_KBdDpVVNvn)6m;1ZG5^laZUl7w zW{Epyxx+x*ig&;T>Z~BIKuvYc8j0|lt|OL~uKxF(j@APowJX^f&OR&kW!!bg)jpqT zpK1OLAp+OP&E5EcPFzdZEfayL%C#FSap2U>9@F{ZFtU>#8mgYVet39D^y{4)vyi?0 zURfEyDMlzVCl~Zm)T}+G&}($BriQedA+c?Ojk(z9gn9nReJfD;lzL?S-_HoRQA(Q2 z$XL^cFx!83nP78JS=nhfQK9QnvWmrQ2Zzw}t6MIADqI;bGc{HH^6}=iYcGyc&94bq z?fH#}i{mk5RbczPAK8WB(Sibkz}>Ig-;|JK8(w{H+P983{^%dT^IlZl3S1j+9u4@6 z_TOP>J^M~y{6d3YrTilLkLzb8tD+-N6i3HazdfdH-{bGw!deb)ZxH)L&|0@K6@FbpS!{~EXrQ4@{4Gse{hM5c^nyihY&LH37;d|!D1iUu^ z;k663gF>C^>)D<^_lydQEq+mrZTfz_HNDRI&m5r`*~^WHN)~bGnys~~=&XH5wBTW{ z8o0Ts>3wie|9fGZxPRl`(GR?PUaR3t+w(bA!m}~fcu*p?b$qf)k8lu7+h}D86t!t- zZ#XB>2u|o#d3%{@&1Zeq&9eA6po!cOULVl4VTq)p1 z_U0X?&LjRxaM$|_n-3@_N5?@oZ(6(WEY_umDSWtDssZ!pnu-!|dkG{77uCIKzo3BO zW9P@~Y{!*N3{Mzp8sOH7%TW{Mv{$Y~N$_KUPE(z77aJ$1AGkjW?pI(Wm}9sY?@NQ~ z{o1_1l4fO)s7{HMr-n3%Q-?vkjGZ3w$P!)P3SsZr7xHT8?WU%t^sqGRi6Rz3CP(c4 zrtmTFX#9Y>ezVLAtt#*4rt9Xm62_LF?||aVyJOW+ zVBmVs{Jm+s*CCI7#ePMaxZ89}1$Jz8^-Hv%`GNe2uu3?cAg^8S^whkfspt+$^QUIe zdp1jgI{cbv)wZ(%y@Y3;)nHB!(YMM<$&|Co3Boqp^D1s`7-HhhKPID)iR>ABiLKl8 zI@}wt^x(aH+kSnlRKC8R6E0u^R9gRXeL}0}3T2yDV^oyPdt!5E|Fe->#vYvo??eAvcbPOYh;ud(MvaU2Vk?n=73DS$Af7P$>=2v zHF|q`u2R#+mRioet-$v6^>uxGRMqq!EC6sD$f};d0C#c`<(94P66V$nvx#i9M6vP0WufY7y<%GzML(W^q`k?Pgyyb?vvLm zwl*kTP7VmLH*5blF#STUgk`A}#*DB0J&H_gTT31pXniHKT+0oG*C;1Zkg!BndQL|A zU3G1!Nz+u6n9bAs?(P7L=}YfrWMZ<8{~bRL(_sti;|9pptc)DbfeC3w34%@*d%x#07>Nb2?%*a z!*%7d5FrCz_Y2y#G@B%SE!j^2^|Bh^aFcmwJSQh_x#woVM`l*mES-Etad8pn&FPH* z`|@KPik~c&v=M)ql9ZA%+Y#ph>@*2g8?wMn$<|Qy-o39Uz~j+a<3JWxF1i_SYoi`D zw9@5l^(}N{Tm2e-aRb0?AR(bOWN~h4>Vc80iL1`g&QkASwq9&(q=k@YtN5oP4CCv-pm(H7H2LEL(qx?05V?tYJGoeBcD@ zk5^iuOC1;POtbf<<`|Uce0^3ss~D!!yP9w!P1v;z$`AQ48t-KZ*9iIdNvPMiUp12t z>Do{z=jCcwX+MAYrf41v=BPwM-q_G!Jziw9of=HRruySJq1u6tZlqo8ZmFG{E4J$g zU=Fv6UQ9wlLTEgF!pFa(+Pr6>cIt)v!3XKX@05y#rw#fZH^WJ1&SLqe$S#rUm)K90 zqY&jf9{FkIu{wJB#+Z4PSV3(wzcXhoPM!J_Ns-xLKE6i~NR>G%9;KFGVPzE@7}(MB zw9vUV;`1>3l+o0mm~;&GMAm3!qYc|Hke710Ki>?mVT#j!8KS(=_zKz8rK71?2|Q5g^;~nw1R?sj$!3|D;V?g*=VWJO=pU$d(NzC zR&nIsRQo0^pc^;J9A@qV_lN*rCMG729(8%3R63hG+M8OU?wX_?=T@x9nzr^Jb_xwK zK)B00MN62rM5x3D1qB(;%nUzmZ#bC^g}quIrQX#Dkl-OtiNBaG+h@Qwx$|bc`z>if z4X72-yR`ZORpPp7>?S|#H!_lwvtGAfhas7#v>Pe0Ug}v~y#8l@>}m#`&F$<6>jvd6 z-sPEVn_*wSeg)jb8$TDh#3=DZN;XvFcoCg`b=%mvkkS{^0 z=Bq>bUe#~T{c>NSmznOF>iQ3pAyFh!o>!MvP*v`8Ew6FsTS7jEsTxfg zvBS-5kVQgFOhBL<_ycNIHoj#huN(>@QDDGE>w|n`Wsmfa+gwc@9nlhQ1>pbY#UQ-+ zZvUfHZ6&7SDNQ-KsNum&knX8k>O?#q|LM~wyDHo|9_JzV<>SXZfPjX8aZU|CmU0e? z0A28iTEp^D_M-HXmJ%*m%NtWY=gH{b`unxVNFHh@z!1QC|8LY5N9)Rg@_&-Z=#trL z_xf^C-+rk=Lw{AulQA(9tk2EPo(?>#ICpTVkGc(#=hm94nJM)yD5N)P?KoNyxM247 z8_WHf{$zAg4qJ6_b2Ya22_H)ZGV{LLch|`OwU#a$?Ue%U8a7W!XVe3RR-SxeTBaA| zasU9sI(kQfH}^L1{XMh1dvKJN8k7(kT57Wrv4m<5c>g}{IIa;s0jVBhs@ae5Vkx$r zSYBI+yUS!c(=Z**sjV7wM-k`nXX%f{20Ln`#LqE-2g}@go*~3J=U4v8M75Gt`=fe6 zH?NmCmm`v}O-sFOK++&3y~LxF_iXK8*J>~;TGUA9k$v7VgTf{N7orQ(OL>+&C*ava z5`5pjP5JiCX1vVV;^|Y@=e`XwC!r})3b9}c@`I7DwO7pCgX9B*r zC~blFOYV2kmvb%Q9XTJG0-7jWBNyfuxD7x57s7xsSM5#L+d)xMUI$OqI=6?0qADt| zH8nM#quHgarT<#kqAhOO=!so@8&cYbKp$S(g;|d+F;R7ZYxpV&fIgs$3^@+;z1K0LmHnlfze9SwP*>7_V{m2*P;u`yn z{s5heu@9ex!}@Ilk^sVGEr#l=3cyQe8iHxxT{KBH`77z~q5ltVSPRlJba!`Gy$Qzb zcI2)sFW04$INGXqd{`5YNa5b}|0NcHi?HX>pwHovnopjLRl&ewV(v>l-~7({Ezv9` zCMN!6E#8xQ=oW1HO(sM3aRqqB>IG_bKu@2;0WW!T2)5DG&7`KMM_1<9GMgSNOt~;W z$HvNf?(De{@GrO>;$Bsi3tU~;9*115c;tbA!#Ua6E!JX$+}zyQ;g4nB$o@4g{4}kH z=>h#2%I!e31jCI)8uIB-j-UPn5?n`4j-z>J#OKd_{o{+Ps~1T~TbtU3K2|XRC&K~V zgrw^@?<(XY2S(fvc9tqV&=cRw6GMf~JHAv@R0vrOFtY>?o_5u77@hlXc8>u4zqldL zBT48O7Z+-@YPlRKBqVhA?b}r4xC$48J%voI_J@kZ7m1I>8ju%$9E749s;WvYOHIs| z?)?#>mCM7!@w9+hh=XI~FDV|`B+{agV`?R4$(6p-@#oP{=V5=QS+r>+PqYqczA&eAf73O;K^i=Z(*wpnEL# z;TPeHmXWF}@2Q6)<|{9jScY!Ot${5g-jERr_km6;Yfq>3vEhPR)7@e_^zI&u+Cx>f zPMw5GDH}z=4u^kxeaPvm%<`3?ZW9lsy#Ztf=;fvMXi4+Y4>o{`ZDcehfqgQu*ZSXV z4XEm6YxAKEWaOfvvldpJ=;qd$y7IcD3Tpfb89;1CwU@;<0y0L3rj*+jGoys{^lY=9 zu+2lYPJs5m`yD0`go8K(gQ~-EJ0-H4)*c6+1KQFQ z&~&B8#i$_!v$LX{obSoh1a#fx5Va}r%yh?mWAE%)XZ2zNDtitD3%yaqr%ye#{;~9u zFE3qYQH%c^7gm|?I#WkXLFD6e?9w)iP-iHl^yOSvwT}>++NGUcwP*p;{)~`!??}ws zl^M+b0;4as{+AX4{Q+ozkMMq}vftlG6g2N}c5$(ciHL~M(%0Xq-Xgm?L(%3Hz83o{ z*Q^1zi>r-}aBy;RwzRI;PTaV8Q>VnfxX2wZ)VXM+zs(Lmx2QC3z^uw^#hj=&=VDOkNrrfa$NidCb<*f|77j;-Y~ zqIRkRin6t^SjtZhS9_|*#8eB`DZ*+e;4JsDbO9Cf50*TpGU`kJ7dZq<{NpGud<6o5 z$jVw>9fkwYTQ$wG`4fxPK#p#7q2<=>bM8OtH&7)i(*Qd!_aSgB)I5DIP^enG0E>tS z8`!RPXGtSa=&z;yzUxfB_cb6&GBen|ok2+Li38kMx0~sX=RT^*&ce+TUxQ&%;IQl* zynLCInVA_;C51{T=a-1I~s*e(qk&HHN(#>_U1DO9S^Sf}V1?6&Y2$A|u3Ma5yi3mwp3)VD4Q9 zbjR9cSJzT+YPNnM2b%}(hXv~F*|Wz;qe_KP?c)z5n}fYkJ385pyp68+JMTNC2%gn7 zey7?pKK-j(|I+zjPKONuC>E5FTYpEd>-lZt!9w^L6UEUyQLD=t<=$Q>#3xi0xEDD} zO->$P#w;Yl3){`m1R(jJd5!=j8>`L)!ptB92Alt}2nbT;4!sT2!Lg~z(bFd-J}{Lh zl|h9~<}_~XQ=`{{W*hni7ckZUR4h&{L5dq-m)zWyUWYFC?mg4jx86HgZPzPq`@&!Q z+Ykw)m@-*Al&6-aL8Zq+jt6*x?9zoO3FjX31g}!t@e$yZ)?fQLw8be;ww=&%E9 z+5JNO$TiIR6wcpAmI!EvtgrX;la~kbGbu6go`c!B*k2hItBv+sbnDz!isXq{9fjr> z?P0bTnwy&ekO-uV-ZZt7h1#hQX{5ksi;ioIAxdY`(`1&fI1dtwRz8FNF!IUTm>TV( z2FmtR{ynwd*wo_PxAyZJB+Mtquoa}4Qcs0=`1wUehkpAcffARRnhLP($hOIf6NcsQ+04ZSC(`}*>_ z{+2KEmptM*HJZFc%QqJ{Cyf<06xz;$5OKJ&va*<%n2eTXe^PYd5A;MV!UYi8!gb*q zX6MeF8~*KDRTaq5QIEZ`Qak*9r*rZ0p}PkgD;qU6C6Fm3+3()HE&87l$1m~=PW2M{ zywu;*WaCOnPQK?BcP#;+h{f5+;ckbymj11GTO%aYK$UNLW_q4i2S6HC#+Y{{`#Qg^ z4mi-H7o*4V;mPOEt5z>uo9fcjgGB6xKNSjaatM6V{e#Jm6Ml$8g8U9SNyV+tyOS9VgQCqM_s>RX>Bdxd~gViC|+#0qH5mbyP2t#-J+WS7j@46opkU)xY1(N z>R2fp*t=w0G*3a|*U2zC@%gqmiWkGUv5pgR0vU3Qvsa=&wV#jPFQ8Jj{%+Dbf9+ME zk*I3J+2G(X0a@Yz2RP7Tce26w_>_@R7@NvSkym3&xUQn2qVR7$U&=G-HE@!Qng_oQ zCE&KcWtNG~7|%dGw>HhI4!9&F`p8a{5J8Am5i-8wOP}0ZsgNdbR;tB*E+1~mZcH79 zqMXDMz%WBi%|dABgw-JIRu2H)-~xih*24w2g*2IpeiuNYvW%ValG(R&hu}~Y08iie z_yC;;C2p}Yav6f_?h1BEsxL4Hj+(}daIfpJ|61H2DD4%Vu373bbb#~>Rwo7cZ)*+ zFcZpA;Jy_RJMZUmzQZ50j|3=~^#R~1k-U%z+}b+ayaO;?#R2CEzn1qM7bGQ58hy=}2vmjMGBD~NV-#~`y>RubCsYt@Q;MOKTPJXknI7@p`O1U#2B;;gwMlHbQ(O$1g)sY++uE}AJkVo`Di{2!9 zX9p13)|*0^3aNREz#ddoR1pzHKL8O#$ho`3LGf2^!17O{chSgFFArc7kucbSg`S?W zwWpa@LlO7rW3z9*Z~cxP^-m6*s`?2CKz@Szk%>UULb%@Rg|v8ke0H>5)^I|UWx!so%KL6 zEb}@Xm?|g`L?&2RSP+wv8UV^|nX?tVKnr-%?_>_tg9b*P7QkQtTQjMx1<2uCtM=;t z{tU3d-Q9l+m>aN>;`MNh#@+iBkIO>LG>6G~k}iV~ki#+!SkH+2UR(6mRsr0+^X!?k z%yn!$dPP})o?Rm!#K^?>o4Ntr+|H$LaNEHg1I_U8%F4=a{`^esT>Nw^FDnb%`Lk#L zXL(cXfcmJ%l_za%6WhKoe9hzn6f*t+R%z=H2mz8FWmf2%<>lpuw+$aZegw4GY)G-U ztaSDM=FD)x(1_p>j8_PTxMgi604@TIv|OefCk80c+_V>knd?|S<`M~5N1c!{HXuFb zke+UTp$JE7kB(T;?eq|u&Gf=;4(Vc#FW$9Gv{-4@{M~_ zRibruPu&=mJQ@-=Jm`#bQ|nwuA;vN79YtwsK0a^eJ7Upu)R3c!hlE7iNuWwbFh688 zW|zwPiqdXy3^@<_xgiR47OD~)T4z3avG_kE6B3f_@%z-kLPbe!H;Rzt=YM@1o)6=| zLty(%yjC;f6Hxk79j5_NCxnNWIv;*GyS;!pSwO5MoRtd7aasD&;TmdVF_HNylZr;I z51(wt4y;o#a&!>3G+hsbE&Vd5Qh289m(THd;1H67nUKWM%@4jrImDP_FrOwb2v^a^?$dU+ywVNc@&g>>OcDORhVM>T}&D z{;dx|64sYUNaF;d9cc;PMOYtGpwsy8F>|MXRT2B=e6px--bj>3Kc5*6Xr8PEG2wsy z{MpZ`eVL9&95&_YsG*^wGlazeH<7*6N=ai222o$T{`^EaEhxH~;O35YcJ&cf$=82}U3*`~YP7#!;?q}eYd)Q?7ifz^r@uz3o|L_l^FG;8}^ z78-4|d*Q0%QesVb?Guvl1JM3>49c8ZQ@JfrY4Y-NSFHYLym~0zGpUy}Ap=}0ZV@gb zOSG@aU|a`fInW=)5QO{OqoqUuYUJeP?CIh(Msi>x93ZOxl#c0s^BaEvE7_f!>6f_&Zb;frx|=2k9<-|k*4v&om(b0Xg_)DVw|0yF%Ve{u#7(XN?wrcaBIN9%N>E6Aw2s8yoOIq#i(T&0j z3X+nOe}^}_VQPbznAoFY9JCW^W7GNL=c{iPN`zV9RI7j>_U{=IBVV4~HVSZlz znbikh&8skc4NyqC2q2`1dG2?KAU%7#hHjVD)z@c3s=QkVvf*YOw=O>`@;GV>sB(jx zJ9mD?t{;UpTS)IsEAwz?tgYQ9xmaX3E?{Ez?qo4R3M&FkV^x}*-$RRbDn`~a{`<5U zF+54(1@I`31MOVH@nRUvbgp}$vNEN246`k4Gx9k+G*r~~OX~0L4y?#*km}l6XGa{k z3b<0mYB2Y#ZZf$L8 z;p6}|`w9&u4qt#K5wQjuI4%*BZl2VJ{0aBHOm6pP(l9VWHstM0hn^>eZUvkf22CcM~PZq{Hud+y9{MhX;!Mv;23One9oTefFx z1JyR~+26F%&g~e@Gtzqa5IC>r)8->A^k5ICkyDmrW0>8wXV$ z3=Kw0{O&&m8Vf%mySsHEugwcvPp)$pi0V1| zh%(1P5um5jeF7)JBgF#-!$$YkC_a4xr1JlnUV$A=%N_HJp^YqW`ZO4v1Ikv;d@;-u zAIX!}Ix_ndpsrOeR{lYgo>S6k5yE0NzS|Q+Ztd2cCK~S^OyaL=6!z8(tWYpR8Sn@*L6Y z%Pkp2<~FZSy=G@;m-q9FEwUyiJywSaDaeU#e7{{42k=Hq@HFNEK)Hnw4h8`-dGW%9 zzi78VLDjR<(ez>|nYjqPB)$IX1$;p1(TM^_Tzq`|Ok)UYqH<-Va3zT(e|C0ueJJ0g zJ4yc3shBx38sT5+a00%o>(TfwZ55#w#F5ECbcdUPpFv!F0S@sJ_Lli{vOz)$3th?t z+OZ?<+ty|V1n|LgX%I=kbP47MPJ~8~(%xEbM!NUd*s8r~s#>g;j?U*QBcK&TA29Q^ zqxC@w$fjgw9=^7tqf!bhp27*;v&aVDX@-&od3Jy6j@UMwvB ztt}(*Kkmynrlj&5xHhsW^*pYvsrlP%)+S>;Ok>hUe%xqCzLGzfE~?NcX{Iw>1I&~X z2S9%m}$fNk6nC6whH4q1iC{bc?_gH7rQno*bWX4 zQ+C}*Y7`y zii*B{W8vVioT~8y#03x+;aSZ-aZKtj!QG^bIuW>cHzT9vNSEK;1$qa8_Ftf7=?j3n z9{0HZ@}T3&N_{pN6f!hcAG59b^tWn)&}P!5M8YzGUh?=vXLZ{h+V0 zE4{El{1YnWC^Vmi_#Pg3*bNSXB;B*hGRjIz*(Dey?z;5=N?N8?E|74Wj}kwK3868l z7AH|^heQMwQUgoFrWXArG7@OC6|p&R3jV9*j12h}XpV|-XS!{98Z(5^ApI(}<~G)| z`%~geoUYs*Mk6DB4q}oZR>nHHwydx`zR)WaLBs!$rw__BNOa4aP#H7Q0V~D>fi6l) zcB&;HybHFRwkyxe%KA)AO~I*nn#T{^^ko>?rG?fFy_+lbgRW%SQ4TWZa#%3SaG80V zsXS=2BZXf9;45Tx#QwEkfl4^W{2^<#b`Ej_!_A10)b_;+MN5A<$sERxo65~N5G4m3 z2EJ2E{a8snQ-2-&Ij;o&rMlr|?v_Ak6q1H3>l~zC|1IF0B-p;L(cw^Yl!%L$M{04o zc0}6+?gu0ZcNs)){nUBaKW&|pC?sT`5#b_0FLX8Tj$mZtyQf|tzGz81Mpp7c=(%6> zC=SX!Ti?#}@WG+Rprv_Atntvy@P)JQJtGK+mg9tyJVbU)$$-JQ@Tr!@Fwvu0cDbD4 z272yFA4wa@cA^d`p*yl~?aolJs_0v}B|ESRxVSP1D7Zf`@7C8NspCr8lQnU+VLUp4 zq=(yEkf!rW?D%;h`jgxg#o`jKBZ{VP*!;7mQaX)k39qGt@LSV1F?zi%=C+u|1o0;Z zNBN^`a3t*X2&<~26ihX)vci4h-s8TnJ3&4#o+A%%&pf#+w$Nn1<^qe9vKo1WW(uK? zDp?;p;9Z1;S|uHJP90|polT-_-r?;IcM8^UAKxa|mUmA$-C60W1z8wgCUt+kk{Ntz zyCcg8dwsdX)z(%a3M`ljQf|VMd>3Z~Rsw>N_3|_rlKw2gVzGSQ@m!hso8;;DD5Ajz zQQgx~QlWfdAG@;-Iej&+v64H21>vTJy>E`J64sL-+;`vW+E<6oWyPDqLK0l}vgIIa zht*@J?Y+PsIjm-Z%>*p3XiI$k%gby~9|c45%zTWc6i7HcV9^i9J9CV)!Bk!$l>!Q7 z(Wd&f-{VEe`2)N5qn6-}yp+ziG>(3@my=@rM>E8%FZbM7aX0D-a55^|Bhw^el!;k} zUZto$nnAWrwhT1z#X#StRgW1BZQnf9$0FcLscPHOgrSJi1se6hwx$|0D-K@=dv4&> zAw3~B6}2L|ChzF0nCB~H$QfOn{dDEb2Oh}17pge8|A;9?d7b$p*?45^|Pt1#_XV@Y~9AS$bYZ0*wyuPtmKdmB;vzfe`+=mZfrxFEKfO z+B!^(TC~d=7>x$EII~8cB#XAS-M`8@aFo~ty%^LzuouW<>CCB2!aJU?UY1+8 zJ7fG<(#&`U1Q&ZvOOfA@MCJApvmEr#Z{P1?6E5m`Z^110+`2CscZUHYGRV>3ic7*r z-aY^8FKR)0h*Z}6&S@@7p;bu@{q}&+y2Y_TK0&yp3~3)W_x%EV-j{?Vra@(x@7AtSEG!vOx9jQ&{fuG#nT> zpAt%DTuBK!s7-0Gwidj;-FHdoPHn^eH~$||R{<5(_VpB@*a_A5cq`O1f zM+yuxl#)tG#~>{o(!$U+gd*MD9l{L5%s0OO``%mYEEa3l%)RHHyU#v5e!soPhK+*N zH8e5u*^%aXsF;_kd=&}4#@1_vr?6C{+KW*G#l*gU#9?)Ti_TNBJbRYNTl0t6w*$`? zp(tm0We0z>ReRt`F8S5&vF~)$qc?aquW#Qyixzen*6dd3l}tXk4V^+@0>po`?l{fN z@2u9AN49}msI=OxH#z?}CGoTvCUA{0EP;pQqK{$3yU)W1H4>^t{Y;EhpGL=>J*sv* zw!F0&gf9kuY2#Q=%PIzb#m0=eq(#T;>}YicdJPXNy_5)xj>nP1b#3Zblg6OvL3&w( z4)U)o^ncXo_@zmlKm~d2lb`GbG8hmGnUcL=t7I;>_s zvhy*Jg;K18GCfXrq~C+E*KI^R)#@4X6WQ9Ie<;D=D#8A9jee#lv*tgOY6C?ByS86W zBCE(*9d*IE);8V^Fqb%kbYmms!9mi$XLht#UT5!yRoB<_t@tI~Xu!zL9xBbr4RcS7 zV1hN_Pg*?79{Gv7qMQ?>2V2Xy#Av5X^z4Kco<#Fz{A9CljBqlN{b@q;Q~BdrDYd9! z4ggapXYhuuR6uh-i3j8Sp2`3l6gj_c>1o+f+8HbRP#SiSC-!&qDPU;MPeLgSG9x|~ z6fJhVmJ?;c+!CE3n_M+Ltx%EdNaBA#%hk=V@}=@~o0PCwKCW)b+S-+PM{|z2IM)65 zRmuNTDP{;2XA2A4DPt%(ZOvA(V|k^#*Am(Xt+CX*@ejKjW4SnLo>o+En?4aYeaK2l zIexCGH`}1eMNLU<6z=jy)i^!<`E*9y-NZ@g_MC6ulVN7S z(^*6G(^|UaWnW_dgHCtgIoczbF`=--%zhAx9FJm?^-Q|c##UtEnPe`!O zU11uR@aDqy<>_c<*?BtD?mzZ@JlSc?JNKEBBYg+yZbTXA#prvI)!B2wDTM&m9&zB` z1NpnJDcgldDyV68hNTAQbzYDvp>5V?&I26g>o$Dhuf7oK&c@^3)K*P7k~x9y#2A6X z?E=%U<>J;iA3SfcWeGLeIo~k7#&+1#o)4SCl(?c$XV4F1HhkQ|_VBA<*Q+6 z<39>^q+)R{aE8cRr>11}B1j+NIW|6gIWK1>D-!OWz_#yI ziuYes*o$Ho32C<;dq){=eB9&gs%LQZs>MVAi*=}o-w8x6k!R7ipAuzU=W3f}&zW$A zvBUmeJ0>8oG_WLQ1(UxzydNd*a%Fo>pbL1x98pfu@W%0<<^8Z?d1GjOf~8fhi{0{s zw|eZp8a8#Un?r)YDBjD`*7Y*X#lECB_7X5O@&ET(8k;)T#} zFZEPYWHHiV`SIQkO6gr5TfkS-fq40}`p-PTnm~9T`y)M*bljGbZ;dMbT*ZNhc=Ux=3 zlbQQ^EEMF+g0@RMO)P=Mkwp!?4o8gdAF z+DT2t(H>1`ogkeM@t7p`aW6A#bglY=*u6luQqg(FC6 zy>662RBL^cKe$QQWNv-XLy=OBb1>kg&@8^bx30ES zB8T8PZe5wORIsyiE%o4^YnYwN@8=t`mR?znh_U;nK8cg&i!v}#8Kvdh-k%jLCkpxe z{n?&cER?IgA3V(|Hvve(6R1{!1im4@ek@())Qo_53$t93roma zh8ad$O$(lid1Q$rH?~epo3{$sFE5@i zm-NUY8fnZVOrLV5kkNPy3R;DpouxGaE_aH`#T#G4wIk4j5X+9cu9Ew)uy9or^!v^7 z5t&T8ou`M;?!akymS@vQ9v0DRwm4TRSe|HV-QnMde05_vLpmJXGjug6hCc}J^HCb$ zJ9sU}H#5^^i?dK>-(xLDDc2B3(u);N=*5D+z#MB`o}i$_!a*iCB1Z9;Iz2oN++6$h z1+=CnudpyL3L7%+uhR6&qScqP+7c*LFFChJnQUGHzle{$hH zZwlzJ>nxL@q7ZRoP1+|J(jJ|mA_&h5_a6Xp;2SCIJ$Wjn7rWN9nMS=mm6}qQ$v0J> z9zmBj&mt)sDj!(}yYO@1EcDYM(i0l;ya5PU-Rd+F2DUW3Gr7xETjZjMvcX6O?Rm~G zeF3YqHDxT3Xdm`xw5Yu?fU$9%jH_pW!6;7}G`~*L2^_kSr_i3;Je%Hovx`Z{F{uOE z>c}=}9*RE~MU^7P7ju67c-yQxBZBF402mw}8(m0E(F-cIj37dt+1-@Hja#!i8uQ6= z{UExG&D&_&7=&goVf}aD^j|m5s`lWJdo;u*FD70nZuzn2(m!RHOxxk99$%#fCbg2f z4P1xM;>?9oetKY>=}_Jg7CFB7-M;Hwdj(loR%eQxAtxqx4G3IWO;ijsN1d!!TXgts zWIhO2C8d4gwt9U5dpW!eTpQJL6>E5sL2Xh!_D=LUpVCdak-eLFyUl577jEPONm)4A zAuLNK*l*w!L$SHY)#n2)Lip2V$FHCEhhrus9rhk~hTMxCGVC*;7M>-I*z7iHueaNB zaS2^x8nS}Jxo9t-FEEX6ST(;N?er9z-@`BG0M@+N-4sVMrI6Xu*+O+^Kc8F%;ZxMx zcfNIdQbijYl`%y5Gm6OG)rrJb%}X<64e}c(BcR=mbQM3ofkNBIJ_RIDy6?6)E&2YM zLhMfG-^MwoZBsjtcOIP?DoW`+TSmknZ#{&3C+*l+M z?NEcUs}JroE#ti`0;}|4IOkN z(dL~7ZfBL?_gy``=!hJ5gCb|r>Kh%3{4{BWJ@m~L=!K0w|4Fksx)0S!bdE3Ov;)`9 zKY!xcdBF4zc%GJK$cGMu&*6^cdW}x%RY90H%P}QWTS@L1V>_Y$WYO6Z@O`Dgp6IQi;t5mHISN`|R{deA>_3J=r1a4Bv{ zs!N&72@8owaWk){;*XtYMMv{`4Sb+8P3qijfzkkkM&zqyz6PcGHB@pU_5E32QoMjIREKcIqO$98s z!J(JLh2+wb zpm%{P5EeFp=3_s(VGA9gL=D33bQq&tNSVeI+xgsQM{10(*j*)w*bSbLV@{P#CS&|$ zzuq#Hvbz^j7dU3E%?>vuf)Im-_fy}!_BXjW!%{M{Z(AT| zlP&q)4Nb*A6Gr@XMaHV8b8B_VfF}L+@oxb?xqmnrd->KCZfjIvHw4Yjxs?5HVIxnT zI=H;R#WT{bOZFNsBlfC|A|f&Hvl$enA*`5)xZmSTIe6X_wcymkX~t@(ju(w(TNnLa z-mSJU^5H{d*>Kt3$*ZOiS5N5i_ch$JB_u{=RGHMZqF22mOUV2$yz8#Vvm~1IW?m&% zWp17VfJX%wGR473^OweAzp-!Cb^bf%fjvcvDi10vE8wdQWUN36@fGi1saIyypoY>_ zxivOzS^6}3!Dk+pZ4OgO27StX4F;kNQvQdFmEMR#;(WV&dbn$u3*g;VFl78X8M5@v z>#WQ`!|2f#&4TjcvQjPWk8PMlW{-koWuxIx#?qgrd-i{CpF!~pQ z&b=-k?=IJ80px-zZ)qyO`hD>`7;xb!q}->n^mgZJ>}+&=)ack)4MM#!8l#z7U4e`KoWW+*84`h=Y_ zZPZ1W``c7`pDV@;7re*oYq%!`eyqT3zmth3Oz_o2-i;mor=B31!Yu2-rTSOP2KgYZKGC#q4hD8}iU1#eriv zAz?6mR)pXqz2siZFI}|!C{1K3sZ1{@n&n8I#PIilU$YC74gUP#eO)sn4F}F2T#?3U z+7FUh;Rh3nI_C5#R`szvgsu==zOAiQ9!BDhQfU-gGoedPg zDk||sdwgt0vz>WjhYA%pr_p#k8#JU`K=Auu?j>q{;yi67gO>UU0t!GA@sW3V)B&>E ze0mx!0`75)ES9?t$U6D8yLCUV9`M?1!Wc4f!K$bq2?;zKDsJ-~u3$?Fb40-LM{VAx zuE%Xh`@{3|WR4@_iz(i>zR3e08ClOy#(9yOFlB=C*w`2tvLou-W&{iFSf(^6Liz21 zs&B<1F`lGP=~|1ptosY#yKcCm3hA*RVq7Kx(^Dz-oRs=eno$wQ!1ez0UiZ)NK;M(i zX|%T|(A3f(c;(1wYI3q&+#9q#wzyE0B@Mc?cEn%MiLEfBUm0Ea&1Y5byik>WeqJ`Y zc@wKbmirvUZvKAD`Z3@GQ!kLb=fkHd<42=r&`;v%Br#3Nagl$|A#GswhE0yvCJO7? z@EX8RtPxWpBgqWP<}hJ5_-d0jiT6mP)I4dji8?BSP|V&J@_;-vnx9UNIqAg|(w(l6 zp{!Elr3Ryn$HQPjfS|Z1yxWt_uHZvF!}fMs%4dGFZ*HBd*fdr$Lu*bIBisVqvYxX6 z)qvZ;o-cuQe0B12ii$nw?_nF18?8Pk$o>n@XX0jz@dK`jqhdGjxzG(B2KbLrY8cN% zvw!%iJgw5?`A!xA>QpYx!$Jx5LGb~MrYcG!&K0woL*F^Dx_f%|59od$FSdF89ByO$ z?s|FiQ zEseLME-fXEitW}Rt9X+-MX#N)K3MJ*uz9hzlu%^{sA?1=M3K`#&K3i#U^1)m80K%= zX^-gn=F{jh$W@gH>KYw9O`xEk*{{qm|5=kIK(lCRy&+b}(a zknK`r$IFRXEe@;AL+ZP%=QJlP*Ylpo66WARI?ANkFPLfmLNR2BEhutXfK2xR>G0v! zBHc0u&pc0CCrX{?Thb=uM48MNm2pgFWyP9UdlxKj!d<5eV(2}Zl{?#IFZ?3*{{B9d zv{WOHm5kuIUg4Yx-G`91xE@2KZ&n?q{tJ@ArqAJ#Nst!}xD8b>-T^v*Ks7X8VUaWJ zLE7Iq9DPB^-vu@#fxFJ*=7^27o2>~=@bdDS4?_oX$Cv%d<-BywOJ^_*;XR#3t!m#o z_d~$`+WLjsn_MALmrM_TXcr3NxG6_OwqDiWEPN!0jZq~#LT>_A%E~>NVYb{sx7Ph( zY0GyNdvEpBqQp5QI&S2;wntzNoI|r44 zSI*Zc#PZmZq)2niTrM0nV!|{UcmNk?x*cHpjKmA!9e!#kU>a~B2$Hlt93-?-(()aR ztOkDrb|q=ApnNd95DNle37(y$5LwBR@ZZkX@c8bwZ=i?u?mHvB7Jy-VS1oSjV}_ZS z+a921<@FU96>*tp7m2Aav%N^`E!d8l_&THM(0Opyd?fdH)mPXy>pIVS_fBAyA$L$UidalH3g@7q2A<8Bo0@8G{`mG7@-~rh!ob8(A ziRd6J%+V6+b;k=1-fP%d-WA_92Y&_-Vm9Wc$XAmP+KO##M`s2h6GLA*+qrvXC|2`< zs*I@yL)LuYiY@R7NTZ|9N?A%0!V)B8ubvnS*q=BUgCZOowqt7l8d@|Fl&L?5X%l>c zWdB$r+4HyB-iCWZ@nQwwbBcEZ<_{^UCnjgVc`pr{2%=@4ODhC=Z=w<5Go%=Df%len zza_X_nUQt2(Za$s?w3AVek?6uBLSdxNBWhQ`#4)wuU|!-IKT_{X`2oF1SWN z7n7unE6Z$MK_4|t9W=E|xgU>vw(D*Z@S^+Pe1S~SJ)@OgJrr3&P*4S|jXK{UE}L(+ zHEw?&x&-iHNwmPxf!={{-t4kLzn@JIt0pRw(7!N7-T*9D>ez0l$Q)(7#y^jTJ((oDK3jIR>=BV>)D{Oc=@X*7*!Cz9oT!rk4TG%UtEl!Ex|2Uu4venuPxT zZO;x1bP3J!1F{ESisViH=YS!E8#dkCFPm>Ffh`SW3(RFGbYQ4=Tn2 za0+!*r;j0Sy5MP^9U4YHtb~Mz=pNv`ylEg&`qz69EY43_QD_Jk@r7P+{*b`1Gz2iM7P#!^m7ur(UHFRgw=Y~?)G zY(R^aX^ohGoOTO(c1*_~-;`gNUse|*mm0|mv|(Xk#A0F8ey8;Aol^SDflL`l@_P5# zo6FVW?P5%!@}>dCo1PIe!1!E_UHB5>`d!73s8e!J!cGZY$4nIsZ{CTX&F(rNxH8* z?+Cc8B(O|Mzk0B1BCKHw49(k4TYI?v)A}6xNzxhicif-LsVR)HNucf**PD#Y*0zWr zn-PyAnP})37#Y{sVKb_0k?-0`0TRM%%L`R_HFo2&Ymx6G%%3}|VQAMgu<6#Vl{BOD z7AYse|bKTdVn1m;ht&T*S}1 z{f{9c6G@PFHy*J3@ofQKJ|6E&MLtn;PHV&=2h+Ie>`~k#SlZ9ZIDKOLfC)I)+kBNd zWT*Pzo0{P#(>=EFEFr%)J!}0ED-~XQ5Uf{m@pn5+$AA)laoBnDCqZ)*4GB;dO#9eK zn>J{x@bbs05YGFEcU;9}cSr5wX+~3Z_hbOs+Z(@5wG+rvm?$naIvlT6={-@8nEyN8 zq&jG<=^GU0*SFk$D!-oQVpNY-9I-aLcA`=fXy# zV-qrH$knAS0WHnLlQXvsb3-u7^40lBPljo0wr200D#MHx-(?JJxU&Er=V2&s8r5z3 zwoYqCpHqyC3^~~QuMQUq9zh38mWc1)Kfm6B{DhOx;!}6+T*uP_CMuaQ&V-~`+4qY4 zvizr#Vx}JjOR#rF7J5P`5%uiH~4S2iL9^Nsam~2BpnD4#}j(*F(m*V zqEk>*dQ=&Q^O-KHXsoYX8rUB2Ylj_5p?8gjuva-k0=AQOVPN+WflIW1v1&S96_Vwhkpk6k6o1+hQ(%S5|>XuNngv~3vM;l6RERHcQY+-KQN?s-*OST{( zWMhAPbPV2_wfxs7?y6h$Ps#v;Ckfejd1b9*YudB`+x|rkZ>Wj)R-sQM0KE5vHB;K7 zS^A~N`0^cA{9_-#H^!ysf+RQ?Q<0Oa|4~Y+f#FQAl8S=a;P0Fkqs24VqX29LjH*W? z+Jb^R-ZpP>*`DTzn^W2j>D7D&`E4Dt(NXo2+?(YGM}Cg*4!oYtZVB@A0J-RXA3T}; zmH9Da{QJ4hOIzfHHuCAyVO6Dkh{YVi;5M-e>B9~0OK%Sdqt%F2BcNJsuEeX>XW*k= zyQOFO1~Ga1q)U(^IXJ6!iZvTxE601wAnv#-QVr6vFyOjC^DtrDrx9c)@}!x()@#~n z;1wTQ7mw8K`M=&109eghzQoQwr34?vl32LHXLTWB#TnE+4|!bEDQ1EvB(mVO$0#z$ z>d5IU6Ui}-NbwaU)xf#8yP1O(z*y0zOZ%b-xZr{FV{!7P2k2XUG7WSzc$;c(dZVkT zsi{?*dJaRdSlJh9dl(kMIg~j@FvYm^boEX34D`(O^z;nOEHM9A)B~i*;^z0Cq*R2v zXDRe_59S*ph=?A9KS%;-GH4k_o>Qjg^=_2c>R7_S+>pz2+oa z;IAq_nk&=Qx;Wy)3VuC+-m{t}QSReX<|-bPOWAM@P@wO~ULn%25rww21k4~r9o`3s zZaKJ#hhr7vN;QRJa|&U*iOd)n36m$MPw(HDQrjj=O&S_G-?EaAF-Wbg=2kQ@qNgO- z`uz?7r85CNtgHpvc6S6KOp>uhA@$!T5ez$DlRBRS_)q1E^D?_=+ zz}Gj@9x3*G5*N!Y68!4q*g~?{V{9?UNN0jY^z|kLYX3k)tK)6qDRGPjk)e;i{c_j( zaOQkWStbSq6EojhR+fqQfWJ^2Gd#arswS7?r==d3lORhHp&0oD12PtOI9|dC`mi<= zriIH1tz=Dg0TY~9vXCI7*P~Z3(YqonikvKpz zd~9;rGvcr{MdEXuFh2}x&rSIBbx5krD}7fDt_?kNzTxVQ%-&_qsaA*7DLWBx<$}L< z{`@z6mUDJRH6q&kK-Ve;?N|GSm%o)mq;n$I#A2n#n$^~9=;i*0S#}c$BB?7?`tkd` z1UKwPzNptZ!g+YiX1UQ`F7P!9S*)$1Ex_Q+{ERQjC|64bR1|_u&jYykc{$9=(vwUTk;p+RxgbG1{ci29P~cjlh(n&@%wBvR`tWvDB6`ceHO-wV2#+ z-3Ndgf3}i7C>XDXZUj~`(Hj6v7W}^7TxvXWcog_LUjr0oK8Z;eH*Zx}m3g>HlNgh& zoatEaK9$9^3Bv>@vb$*YH+Blorz9B~gfttBq!cS3>-@qJ&eK)*VsPto+Rc&MU+7dx zGObu^TBlmDCTU(>ubo+PNgQcOS}5yL@Y>3lt&WAX*9n;SFT z>Ezxh%_a_rvp&6lm`)QVe7Fv4|6;QO`hC{lcOfp2tD0Iep75Fq6sYZ{yYKuJZM~g5 z%aJ~HH~9_h^GZje@2T82fsk;0cEb+^3Z8*#h6-huTG_n*j9fVq=*RP(ZUrmwHW&iF z`?Xar-ky5b=XwH8{7fD+qm^bZW)s&#^DR?_kJbR$yMh+E&N5iXtB`YMByU-cyjpR#h! zbKR4rMLY9EUoZRJlV3WTYdmTK~)HBihH9JeF z5C;hR^h4Xn{$WUWl9wtdg9?z{f))3=XZ&bc-NKa>Ay{1|xj0L>I@|{4y(lq5o!0y) z?0peQwgry&0uI5u8|BO`%x2ugaigLLWhOv&cB~n4ZcIGrJ#pCwXr&v;2;9!9Y*NB( zWOwwYRp<@&KVSx6kEbW_Br&MAoZmEB{;dv)Z&G4j^1CedPxe{-!DqY;+j=3|cxHwz zSamP8?s7;Hwv+GEU#kDm8;jSDW6R6`c1{47T>Xjo>x{@W5?hwR0_5C18FO-R+o|?a zqxBF0LneZ(F!at9%ty#X3JO`sKwUbHbCdUH)3XdwE7*yfEa{yOp#Ijk$>O^E7kWOY zr91_>s0(Ni^&JPm6?~PiIA$!KZFdX57Ji*Nkd(UuzG~;VNF65V+bN*8f{d%C_KgA$ zf}508R1ruNp;a?r;w1L+mW@rCQSg9;&h3wNvDN7!>)lzG!(lh(f^BBhW1}vq8ZxP9 z0mQ;HOXcA1nv&G%0VIuo`U)gl(N*kRV=600Y-Ru^^$p@$kp0*joQMKDei$1OdQseJ zvH9JnWcSF3TZdlyD$i5p#%*!>GTeG;?2$w;CccC_p#5}Is>XFAb2N6xsmjtYvk83+ zy<6D0$^{|F96NH+(bivG$x^`fjlQ+Xq7Xuzav77qE|dI30pvIr^+>xTFuxE9RO=#{ zY6F72T6*vxQbf$5a&Q(oHNy)DL@m`M)WixtF1y)}qPL!#kG+23I1>dU!(M#cr7|Mevbu;wIuCrLYQ|_qm!LU%_`Wiy@h>3!UQ5&`8@;BJ$Pts+jr)Z+Zkc0%SiIA(>rAcQ8Lvd%$%Sr*Hc?&G0SpU5^V7dO0VkoVUE~`q=NveI{X> zczENAlEUm^g4KtV2TjOaa&0$rHxp-g$d{ddF7||b_|I2(B=9zFB)$T7BJedB%U!H> zj7c>qY?qbXU@^F~`f=0wF%7UZ9X53>!4E4j+M(*drQjokvw~5xV+#!hjjj0@r;BFU zCsN?U@L%lJGsKRp8W=};D>hD6A7eOKQYI%%##ot{KfDn!8;1O-)l-Ok)x%|V)?WaX zuJJaLAFC1x%=Ja=VY;CYPZHVg&TEMbCIQn2hf;8NnwXcmIU5Tb^OrhCepN6K%0qge zBY%Z6QB#WrzJb10L==%Rox;#T9>?n`RDzF-;r8l$#c4ScF#( zA~`tO&xA;XlBnz=Nq5@3?KZH6L`cbZV=Luo7U3bP&3y`FKZyU2)lROgD#xq-7 z2poz56ZHR={!8$z{gq_z9eX#~fhWY?fNP$EQ-{KN9^1em>D>%|d0Cv&dg$^b zq@=fYt2joCYEr^GdenW5s#DWX=#(J-3FR7DId{2nqxIhPMGsqdm$`xER18$q4weWs zF0{=xJP&8z@#xc5i{XlmA&%9!9|M*Hg;FwD0J>++BR@o%C~18h9VmfJMaKR&h|7~$u9jgO$g#{3mBFJQ)1=(hr5IGaguNd^zx$l093_o12|0CABvGpl4|3iN6r`k*)kIySRr@wS5(+@vl ziil_dm;rC~LEhfp@-{?N6G>F&h`->1X&}i$cC2rpgE2#7e}C~&n-_MQ;>-M=nF}sS ze>KqImj(WO0P+rlIJuTWoFpXSog&;#0HxqZlZ;wudssH@i;f(r;t&ZtJIC^kwxAQ# z_b^sbG{M%5FmiQGYQrjvtYy82n&QhXRo;iP^V7JKVd%qO(HH*#^Qcz5N(;e4%v?8X zoUi4=Qmv9N2X$>~`E>>tp%g>+KOLms}rmlYPu z#6TB3_t6OFW_2pwQ5&*kFW!a;p6zRB3`aoHlQi>255x=%41>O@Oa4C=Kmv*RELh~4 zF68na*Rk;mf)rk~Lwz0@@u67zF>LgA7CNxLwQimttmpl!;}ExUli=nJB6QsqZntjh zp2sYRzP<8(3zpwl>o-JvtticfFKgnr{@yAMy}c8N)9cpW1YU4s z-2)iCsoxeP_~IR@AIZc>ft__u{E`6_`56xS(^Ur{nIuAXKE_Ys(xz*xh$26ltppqD9-1l0qxW5Q2ii zIzgqfvSO`vk@l(po!?WY03+>|5IH6=%J;Q%iy4XxaNC++);gZCgwdcA3ZN~fzfh;x z;ugVw9eq(GL!b4X4?jsn@AKHvPQyIGja;|Zymy%6{9s!VzEZFYQAX3h=+NtrXZ;Zg z5qx26ygq1UVsSA71x!xA>XN{uXrgXHk!sC4=E7v9G=qC2cob0L?6xoB+2j!=R(%pso%vR^o#&Jwp~w!xt@SB;{xv;MrL#8J$-PK30oy zeEQW<^Oi)I*d-)*7)~XaQ{z1P;VA1FAaJ9=glYWcEA>y>ah`A7O21Y}BzRlzH|SNh zZn@aX26fI2Ijbwb=YXR7PR3_lDM}s6^_ey$ATAgB60i zUNG0ZStn5sEV;$t+Qv4;;BA}ZlvB6z+ z?07LM>hz1AfIg=VS_e4GLFKP`vPSd<8xp`c~DhVi{=BRDU~A;#bQ3 z;->o6b5Furtr_sl|8^UBm}5hMs#h)Yy!8|A?BzSZW)S{eF$HU?G(Z1sN7m0?%-B#x!Y&rQs?@gAGKfK^zc{&DsgcJDk#9te>2S{PrWT|ylqjWl=740 zzhasM@>s|4!?LT!^g^=djOnE@HRo}TwpL+VA9YO}9Bp;QIqe(yY#R+a`200Qk&VOX zTPR>YIeBI;?sM~QY1#g~g~Y9h|5l*P^rPgl$n6rc6HI=VxMbYK+|oeQ_J@RBBt(3x zJ8{8(>-*W+Vce+Hm`wRj$M^gPX8x%L`j)~xcNZDWem+qD~mSlJ5W3G&W>+k)GM@?$n+#!rPR2z1@<*pa8Fh6^T6;RLQLN$c-T;p?- z_X!Fjd#Z|qsd9#?pc$*doAvXsR+?tIn?l=FYe?HwRjN-)pgl|NDEw-%7X(+?^p6^ zF9{$iw!deDMxKFXP1^0Jwi|TmI6^Sdi*edXGE-2nPkcVpmG(UcJmyNfeQneVzsPb{ zqhkPmC$1_<>uz1N>VC;V#?rO@kwo)}oaHgLQ(T0qlKBf?J1w`{Fh|$60$z4 zq+MLbE&xA7zK6iVXRjCIiW4y2PAv$w=QMB}WVOUbQ&3VrdwbJKcihb4>N=gdB^`>zy`!(zcbGwe=Qx@(V2$G=JD4vhA^ut)3;K3|IrLBv6Eh z1t7DfWqn3tiTK(Qa!Ht5@&b8(Hfyq+E6!DqiVcuZ>q^bln4KFqH9J>EGY_)J zJOxO1iytGTrbi$RzH^xHeLe;tK|{ISwwA~R4ZLevJj$9D%pR4C690C+`kH3xygoR? ztTDI^&wh^GAIuTI)w17(K6vi5o%_+1`1)w6$<9?1IB}{`MeDzDR4x6ECgd%|D)2H; zAu}uMnGt`3`%v?0d^CeE{6T>LG&KOOo;IJe z3Eq7|skb=EIR#&3lh>LYK$?Z%+v99sv${Mxto27--Unfc)?3iHDg={#G#bP<_X3$A2Sn25*b8JlcY z{bw7k4(k^t)CKR6Yy^-C2pD-hX9So}YPhzx`C1gFu|F+Qsa-8VI7o)&8~LQ}_aT*3 z)W=VUbCq`ERScdjfx1nvoGUEScDYuuED)jZG2qtKAdzjbkX=`{r(yI1lT;@iv@F?D zaB#c;L;Me@rn7B9E*BWsICv|3W!axN)DpPgbzMA;Uah}(Aa-KjDEf=(0sS=vJI}S|ov-Mx@ya5(x%vNd4OW zx&P4x1(@2vV8k=Vy^^`MxF_lSKB_c}uSR+HiFI(Hb&jVNLRmCPSZ`(O`)O) zBL4`vv7>aOAw~neyUOk7 zTWA?vtf^S}xVgqws=JSAs;&jX8UlZVJzc&<2 zs7?*3tb!dY5A7|)#J}Nci_@vHk%Ec5rH12`#BgjQ@%8;LIJimm8T*a1n#?$J`?6;` zJZx0V*CzhWt=qVd#N;ecoBOidaO#SPh~|ihcU}+u@Bq@R-!Be*S<`KldP^dsV{AnB!pyQOl4e}y4uF_=Aj z_OIT=Tw18f&37T4em_1j^^m=iRa17xi&9MR_!8QwJ_x;}{*QdncRg&Rr0V*r0@X0j z%}S0VkvVDEscBE)M(Sg)4k{Ih<`DnD*W#DTZ!l#OKq)!L-jw$9nq9BxS>pVk7g{9R zrfI1>rlWzOu+w@4w?6d8CR+Hy>e{{^qTi=tiZ#;8T7`>Y*8x(HGITSTkmgf zlJfNym^S1eY!)UKB4@A2__?Z{g9a;t3I<)}eDW`DOQgdn*}!WXG-OmpEUtc3)%HVZ zDxH?OB{x-G%>IbDht;-S`Xg5AfwO#Fy7HMrIs1o-)FNT#KE}PjnLjLAQB%<*JWb?1 zj={0|AIfrEVvW8u3w*C(;2zvXnP@RdHm4OA!X!R`U2d3U@945J-0IY)z7?z|b1t6O zi;PKYJcwFNh02^~%%$$g)yzKsTpw{hYmGl=O-?Ar$C|Y{p>$Shoxq_kib0-`|20YJ z$sfH~KlRW42>(PzJXz5#&U~t}oNJAGbZWjC_TQtCbr?0V(7dc28l60${jsHQgp1!| zi;a{1SS-<+k34x4>zk(&E5D6~tne%*^V9m%O62cQW#x_aZXmaD_gkFw4Cb`nXS4Tg z*A)9U_FWBs!|DXF`9yW2N7YBkK7jXcXqJH5CN|4oGM{)xi3>>Z3pCiBu&{M^1KpKT ze#2b&wfkYlV~%fv3{!~>oxdp>wKQZ;Stn{#SY`{DF$aiM<^Mk$@a;KvBSQ$V07&vV z9NXtzNsti(+bQczV=6q&?dRE-e|c+d>p?m3n0CI0`L~W+6S&GcQ?Zn4X@)t?a(-+U zn)i&K%wehXsL)t-;x%SOsV%*aBDB|Fh$L*ZPftb4yCi1*xsXWh(}-tM8utjT9WvkE zAfA|N8T{2msEC!UZ*TwP=aj)TPMyLN6wf9Zbkju?8xv4aJjeKs*6Pwts9hcAMpB1f z5*)WR+`{1RrmN~Y)7m3=bt$~~=|hyS=l*>y+Sp+1Ej28z|T&h9#b6=f4jK}e7uGTj|+Y%Ml7A|_&dmcg(46OXD3Cw4@P z${IJs{f|-0f1~t?=jERAoi%>AyMbcL<9SvS`OXSQids|W2b>w?ILfb={@_I zM{6a2xV#wOoHn1)UG-7goGm4fTq}df9J9$n5A~ewU0F1+#=YkMHwuSUL*o`|XBdig z|1faKF&cjjiCjZ!ST3%o{rd`}*39-}Q*U90MmuGy_?@e%M?_f800B^rb@*e=Lm%!U zh6ck?)-1V~gd8l2V?o1`k3+Bw&v)@C|3+kyGR0`yMk?)}$>HG->b`%4pB^pmH6$gR ze$|RiE?0OoBd;!_0vrF6{6X4niMZZ2L^&`S>+XG%C%LN-bYDeiMxvBnK8_rp>_*>@ z`Hy|KdFm_2rGM4qEMSi{kP7}vJxukV=P%f?rLQqJ*VflT8u};-XGuzde1ZMMjU3An zhp@MBI@lCzp_R&P{~5NheH)?b)7FH{co$_p;fO!aOmM@z3(Fgn&Bqy@fPA}G4p-`B zO_D6V%&W36f(=hjJh!dU3;iFbbR)K!BV?$N$eKLeVT)5ZSw*D2NM{0n{yc-$E!MQ2FiEQt_RxXs*K z%YVYeOc6j^UvY2NEnE7gwLpI)Jk7QIUr}h=Fndz5=m#@`El-je$|RL1ZU&5)ayxQ{ z7{m3pw@l;&-^g(U6BIs{wl{m!Y?2?hsJHnKTlb9$%5NcmrQq#NM zm&xJcSxQt|g(HG#EOw&bwj7w+z5Wy?I`uzPasE#J0A$XE%!xf-=oQfT;T zxg}$EAn0#$C+3g|k0&!Y(LbZhgNIsRpQ%(@9-`#-2?!$Iisn;~{%3#^uiQUm$lEh` z%4}q}HD!X$)5)4zNitECAxC6jWS0JgN1G!64xv|5M!Es+CrEw`n)csWR&M6fJI3%f@e0sg6V&+pEK5EgB4`C4po6wl z*X>#_On~VywevW?rVn@156ToZ+~EViG7}MwP?7jvo6i-OP-?I+$YNxjNd#h@OxPCg`?DL-6k5fLYE0$ zw6LR*tjqyz2ejp9>>lYggba(OS|0;V6s3zaPe|)q^DIg&+8kHC& zAzcCjq9D@Ul8Q8JAW|ZNpoEkGqSDw0%s{kW$;kDdGRs>LQtkOD^u%oKEe@ ziMKA{q6YjvmZh&H$655?4jxNGqW530;e0$-lK?cHyrQcQ2VD_p3mU{@h%Md=^6Zb8tAR{VT? zYl9b|fF3A*-xuS5D!tpAAJGz)zfYnPZ@;?M4-K^F`C7|CnZzdJ-t}?ymk~5-^B5>) zKR##34lg4X!#&9U1vxnSMCwO+Rky*Ovii~Ek3R|nX;gHlE&;miNiB#)(kyOQCXA&U z&F>lVJVCk_VYPv{qOu>3+Y1aYb@=mS#;~$lKn)e>x-Ksqjj9=V$q$gyOQ8lAvo+% znI3Dwz!v!~+Vs05Q>Rdf7F%(tBZkNgdVdmJyPIxvNP{>p%+5SodF+|n5M;Fzv_sG3 zm_p(D;86i#Q_c?DJ@Gt_LAOCiSRcQe6O9IY%n(4L6l z(4`Si*h@SeRQXUWR*bd%dz#x^uG%xuRL{%Re&`zKhn4?Qrli~Mblliy=OIT=ynm?e z=GNOD7I~?_is#q;)`w9f{d*N1W4wA6v6Dcauho+O(U%Kq z2%Zl*R1G;w!zo}yX;^eAEa~%&s;~-SRT&Yjj*wIQ>eT4EoGr=EOf1lZx_Hj4#WWU& zJ6u)-Ml97G#WA?MyPppZA|jdti#u7fJNnb04K2IHF(PK=aPU03F>t%#n=YJqj0QxC z|5@_AboO_$Je$J_>vj(`YJ>kvTr;(;&+-1EBinA_|HUUA9sDPqOEKBVHC8ZymCyBU zN==t@lvgA>ZMw)4-c7A65rIeW+R5k ziEn)XBMm z4=(5FhJ}Ys=o(_76|z~kG!5a{@B|KwuxqYQkOe#m5Fdi^zrM4+!R~xHTYQRpB(az; zIl3&O_%ZDURQCPqq>--qR>4#L5A*q)=M ztfZ*A4Xhy1ulB;})G?@3rA; zk9e`fB)VB?R|HW2)HTW0I^Emcx4D}POndi100}}r`(4J>Jjx+!zqQ3%x~6IkIp1T2 z^CbtZ%`6YHhz>5K4Ytp%vr|xAzVv)Up>-U@z0(H`+50-F|2TvB+VrEzz^(a8{j-rj ztVKG#TJ>Y26lpzexD`bjD7tCdqVNtt%gPk$PgG{cW--S1}T z3krTtul7|wdu(zKvnF>N#!l0AoZ_~0-mkL>Q#$H~PE1c(pQ$Fh`C0(w+eq39g_eXy z%;iP=PmtTl)Nc)46XMEfT8@Cb2rwU^iNbDobS`k?<5jmLdFXp=Qc@f=^u!{$7v_6> zo&+BnhBw1V_skn>lvIV9k=$%QAP#5RO*)QX|kILhjOaVMkhmeb(t z#zA@(W}TODr`uB@?F!05Um2KCa$bw5?w{*iCMNqW4%=^$1PpHUh(^CWCr8EqH1#v` zGN9QXnHPLY=lUu!0ll6(Est~q-oQFf{cL%$vNASM^RB^m;f|fW>w{6BT&q8uuM$TI z#5dXf(-%XBoz2xtA)Ukmd!ZM@tIPl0H zf_0m!vsrHQgO1kwtfODQ-)X~d@7SrJ2Z=?$6nDMChhD`D`Uko(a7ov^$7WCYn4g{? zdB4OlbBmf+x>HE1^({2}!WScG&22?+E0sJI-l~KN=u||uI*(le=#EM#?ni)Uda0$1 z+j2P`(ILK|*u1V9bO7j4w3_xcr3r~PC8D(arxza!dBN`IZzWB_uCV3mA@-<`oJ}+L zE`JiUlX&nnta)`4(f_0L#?HiM&bOK1$5bU0kjadES6^GzZRAdKUWCtS`Ux$iWVL^| z1sCMAHV|4F-V>Mi>(|!#%Uv;!Q=bv@lgVSfpn19ApMpGEJzd_GQUv?cGNO)Dp9-07 zGRwD-MRrSW%||n zZ!SP~J{!=(W80IdqG~5Ten{xv8nIZy@k%7})iyCZO|rr1$rvukR*bYo zI$|FR$FzL^uArnOBc|ZcnZ7VM?Snh0rhqmRikN!opY9Ze71hw+qz;KGoW&e92mHBG zgOpFFI4NTa$*nKrFfr10XY~y0 zOObR67X5%IlEKGDAvnU`LP$7gcb41%e7wo^?%0+}cwbyNX?V7AE1L6UL?ire;AjPNUIPnVht>1M&=EyOqMN`ZOJBD8Tn6 zUsywux@Ih$!guJ_XyR03yXFN80UqVsIFoxi1FsnA>$jbNCEeF*}^dVDJ; zER1qn*FRan5dsOx+~`r6ZBa@!2f(({HNnKoy(!eZH0Bbx>A8`yM_% ztxGNVw+r{$DfV8un~^=(KC3kzmrZFY|9}!A|LQ#dSX8GY!rw`!{_B8 zBi+ACO5V^P^~@!~ac5Mv<_(T(zoL#n%?&&tm57yZW*Mase@<@ax1m%0u;pTQo^v6K zL(~x>11!q8Be`i`)D#r|WTGNU6^}RTSPKz2L@Wj^;dSy7w5q@d!=Rx|nGbzV8->M- z_)Q@G42qxDWCC%rjJ}Ck1wLJDKVtw!c&^qF+qwkK;>y84P@l*YF{v`k`yAFHtLeFV zgh67?Iyi!pPaW-tuM#fEIx-VSxga%)8 zg8E2T!xhr{r)0XSWDahcV#PS>gWEqkL@Q!U!x)tC?W?UKp>RJ~Qi96QwxI>j63_Eq z2;qM6qH=fg=K2DD54K7XqLci{7mTqHO4&4QcD_t9$STgz3m=<7j?A4LV$b_%bPB@$ zyww#DQ|5YkTvOvNAi|jB>g+1f9?Pm!2lxn@e`=Kvjt~HYlyM(MZA})dy{1oXOv8#( zSAY`FdDNQg+NR&1L4po(q+&V?>AVs%)h?M!`_4PzjQKVl>EUeRl^kIILs&w=_dke* zmb^?fE!Whnui|g?kP$*CxePU@T3A@rpTgD|%Y;;KuVh$+;zyb(Utia^y&hSj>opJ{ z$I$97)s7GiWv3063Ei-km;!|;a6&xwSKkB_`y-K5$$3j59$9a{W#n~ujS>5cf2kbO z6;)Q&40TA%55LlN1jUwCwY_JAW~9p=^^CHvepsgNMYW#8ZyStF>AG584 z`faFaJ0-#ZhhiWC-V!mIHY$pRs-kS*i^?h?-Ou52fNCP18#i8gOxr0)f5@Up*$gr1 zO+Fteot8Mj77q&Ii$U3!5A@4R@4X_tFZ6vO;k%)ge3K=~*~R6myt0@wOM({v+maH4 z1cICq_Iu}n*z|C2$f0>8K$H8cA!o#&V`^bbn4M+SotyZ#n+)Vm!#zSSjHhaHn51gq%}Bk<>HI1yjwyJFtshfF;#_#s zxV}3e^@RH71+R2Yo1P_djJG1dr^g64_9y1^AJbA)?8v@!US5p`pxEz2QNiRr2LwBhE!Nm{X1Err;?nRhYWK7t*^w@MeL; zfX)t#RP4vvy{@ijE78uPaco-l7Q^E5RqrV=?{!|G=x*OO-~`?JvkS!@;NswLxlL)i zTl=#*;m}k=B>Rm$vst&k0|Cg^dB43cV(NsEFf(hGvgvK!*7yAR`Jt~xZOfNZ@0oWN z@A8BW6Mgl+Btk1s-TiI6eZYF%DMx9iM&-PlQ4b>QA;#Fl&yGSe6BEzZaq!#-t9ftS z?f~eL=z7M|6-TI(7X(_tYmr~-8-GQnp`;D%wgprOrR`cTjLF5Rv<`wp5V>5j2o$PQ z4zg#bLP!b&V_`*(Z~Myed)e1o2bd!GMloH5CTUoY)n3Nn@wzkO$c_X_*RGUT-0QHj zU3gm<6iA5DoXoDwRXVSV^35c|=DQZGmq5g&BjM123noNlfOFN&lIQaytULmLCe zzBTjAP$ebn4H}8xOaJa%*L*2mP%AGYl$6Kc(w-wibDxSY-IkvOKL-tkerk(iS2(o3 zcbpGi`xAlKgDy{T+@d^v50}Jfnv)Bh^V}_8Y`ueVQ66Obrueuwl1g0k=5<-uE_ChA zbh(Y{6Ri8y$D%gWKi4eMIuD!6)v-DZTok-eiE{L>w>{{uUx~jxvFMA}9$fR8^sk9u zO9bV9JaHg81+i<@7LXyZ0|)mGS0ueG(K*sXQC&d)N>tkU341jca{yC+iF;aA>(o6B z0juisLw*X}*Pc5Pj)yG*5ZH%nzO7j7lH_XdaO=RC-hR>z)E&D}8NpU`WXc-r`!mOt zuT7TyBDU62erpX}zU7ORJ|`(O!uR|*;O1KGeI6O-P&>@g?5)KfyRh>sWV0xWT4Jkx z-xA*^C8~=L;A0Qq^X3p|xDTM0V}}SL8R7c|!*l^!HbI0SPoSm;&A~jkIR;-{R$BVe zXaYWLR&8l;pG+CJoIsd#@g;2t-{|52S80oLLM6iwbm^j$Kj((gPu!N7nv|QAlbxQH z^O-x%YNk4wCOt8jEZyLfY0&z&~K^-GDim&K7^sq0~>UX&joFroHH=7r5c#FU_ zd!Ec)N>bsrnWyN33OmcVb?Ykc^~8;hO~Ji)X`vT~(c6eWy}zBuI&t9Mkw(W&Sg~ANn_pi{T4qCK zS>RbuKw-qKK{Nw0k)~h>bE3f1DZ^PzdC-x!r;0%q{k(lgI)vNhuG4QUDXJQIX7pIJ zXNjq_w;72?w5h;OA4*jPIs4OrB-NN{U!nPb?9+SJ)kvptJ>}CSR?^KScvT5ja>SV0 z;TX<1u73bH>+tD;ZA=F15qQ()Kj#oA7F>=r-dwHD&%vM+_wjLI;FDcnM3VNG{a2*S zEM0$`!GWU2R`8hzaC6H?fP-~uVU>feYLJ)(?mbK;>uy)ONTOL$m`8>I2SYtI%u##! zeCPUgA}?cmzQEcxDADnhjw#ap@m;-#D#H8uIqvVGR>g$B(I6lvCL=~lDskR+Yr(_5 zJ}O{>h&}0a%5&a6-FpG&^Xd8Y>u)Ka*cEGEQ;+>7>v*#Y04VA8 zsE63S6!JEj48rY^2^3nC2$^9KBg|jCvl2c zU|6arpt5yEcMxruTI}cW_Q*_H%WNw}jJNO|fR>}AMR(U1&~+Pg>+_KxK_9lW9UB#h zwzDi(_~CBiO4)b*}0~dayntYigdV+NqZC>s= zTs2I}dYg%up5*lLC>iV*3r51*Hd*-qCzCptS~j{j>q;zDPMSt(xx=n@1}0isefZrk z$Hm3X%g-2P(;MA;e@N`s;^`EJnBhs~?)ipXdetAc+c;woGZaJJj~k->svAh3dHG(L zLEGtu`F$UO^tAVBT*D&?UC*i6BEfN-HsorkBeJd7?q`AUQ zs(y=HiU*b?M6HJZ==Ji_2AyPwZSjp-YNil!*=O6j5yY0Gqa$xf+}*uM7_oxBEvE39 z;7;tUa@f0>(-`%SBoSfOvJpRz%EhfJ16Ku2dQc}u`uaP1xGAPRz~{H|=h~=76X7?O z=jgdfPdUTuK^G-jIi*B zv^1{KH(v`2`+vIy6Pu6RCX4T_{vC4$MStI*I)P{~-A_4!jiGfDqfJ^>N(e3#^pAi1 zEeyfi=YJm8Wk*HXOFUX+=Yh0ZIrJKn#b4qX?ICk)C*w$UdgRKVu`d~|hX$rQ7QeE9RNe{ z7G@+QB>Ys->!H0&H>dmE)iq>D?=t%Be6Xgsp}%H??bGjw!H@nX3zf04mh5Yef#Wif z=r6ztQ{a;_RiUpr@bAScW3#^Ufv=yH_dy+WP(wf)QHYaFi@7Bb}D z$<{nf6IKPrt&WO_lS#(+41J*4!32=qZ-Q7}-=5Kw5l_o*+PYv+)}ZtkJ!92JK{L@< zu##?2=K7TASn}zvNr(9di%wZ)NWr7fxiTi|y8P`YJEZU0(V8vjJh}KU))1UXB}Fd7sup z2MOuFE+L+;(9dnz6z9gSIkt;BbaO7ac5TfJSezZc9AC9Us^GmdP^^sBn^Uc&8>U^` z+#srX7e-wQlMOk{E=|dWZmCFV5TBv!_pesXd^Da*%oOBpelUkfp_3b1F_t0|{ZnXU z1c*+EPf9Ws|M;$xG*FdUawe^poKdu&+kWlilxt5UT_i27amOfVsC>@KG~!$rPJ#<3 zjG_F+Tc6RxI?CIj7^^ccSj}EpJ{bB;7b1IlMkc}Z7-SK0l5~`YHT3nR{8d!tE01T- zFvN^Ajh36A_w+sG4{B@nT+`-XMLi{C&^&N`EW}h_Uu!}1xN8eia6dd*{-Q$0;R$Wk7oHTv0a+_goD1_Y^b~ko~};HIrT#jV&f#~ z_GBZ4wgaJ+&XeTbN%8S2iA3P(`XCDy+&N}Onapli>plKalBw#5=I3FN+g+09HT5w> z@Z3DSw1!aHfMqrtQ6+jSVw9yWEU4Wf!b2 z&p~ySl-k$QR>N>re4dN;%*dI2tBuo#?ZQ84DlddFqvjec>w43I7vr8 zD94bHxZno(OVQpg&VSN29U^iUpi9OjDtyiitL5p>$)f9*r{Or=om%5Nh`1ci$Buqy zL@COpskai249_>|28MU=02J%@?{LArvzPSU(Dsv`R8^V)pKfX<5~_kn%UUaUXJ!&u zV^=|<;@4#Z&)<|PvM5zWcBHu(>Ag~g@+7mzEvj<9QB~&m-_G%vUpzbjar=Dpy@bd= ztvihf`;odLCqe9fMdolo5q+UY+*i@#PVE9~*0rS|_CCIPtNDadNl}9=vLA=5cwNc3 z8!ip?in5GyIfKA!SRvsVxRByfS zDOH`XEp{zB)}!lKLGN&rnVHe=7)Q2zK0Yq3Gm_(J&`ZOBEoiKiZGjksVLr7Y1iKkF zTCHTIzO7)d{CtmXMB`a4*8q-L?KGR>ZcMF@d__QM3$^Tp6nj|(+a?M_i zD0kpM$r{F U`V)F$2b;Ok90+qN^DrUL{PhQU|)QAEYk@9tp83qm}g!R{T4II0L} z*{45kI{6f<0}5$B55wM_CJcVflmC6XYNv|AfBKSP0L;kSh*<@R_whfN?BM6(65=ET zm<<7r%E5%)vQ~ufUcQ*(S-_$y5fap&A9hj&U9gXB)L2*FP(;`bs`j4%#y1SnF@9pY z6*;?(Rsx%E@5Au7R|+eWHJz<%gWTxik!EUdus5(hds}s&fh@| z(7#^0>zXh5mMfc=l!LPWHc%?r@LA2E{59-RZ2qWuh4=XBIMU|B2%fIqhOp~2T0*qE z*HZ@EMKP`3&4FGE%U2hIcaqpG{vZ=ia27|dKEeK-9OYu8BmLaUU}j!kJTEJ)A;0f% z)nrc%D+}wjj122L1V+*bWc74wqt|dxy)5{s&XeF!ZvwWlpyi_@Os|Ho=4-ARJgw*9`|oUro1B51f|0TKSfp%KnNZzrv%NdsSDA zDa`YYe<9?&j#G<(i_HO@9_Xpn_6&zo8cO5qc@vtP@ zXQ68Qof8SNvex+J>tT1(^ixT6cth5|Jk(lWP;As_G+yg2Ex z@%p{FFzk7PyTJIFPM+Mswt`7sn9Hmoafai-%ipbG%+<>gRPwRMWs_jOO_f_(!qpweJBN z!#SDEkPwfD$yJ}DftL? zKA?FbBGq@dPGCL~_UH)9${H3lG60XKCIjkp=#J^^rRNve&A&bPSlVItrKC+b zA&4X2-uXwQd!SmH7dw@7n(~b6%}!drXGfRW>>EMM@w`H7?77bl=o&x1!p`xmV{OUe z?pFK5;gzMI>h2VM+~N<12GU}9u3Sq$%kx787!l5?ILt)T&1vop3R^$jJKphq`ibh0 zfn8dq{98d8x{rc&RClm%sp~v3^s#jyY%eiC*!^gGxD^9giy9V2LhG+|t-L-jF1CcGay!>nFX&ReNC{YyJjj&ZYunWgnyWZ!q0+R>{ zxw-}$Q&)HgChYo~6)1Yc>?pnBvaCgUBI$O!j87&B?yAz#tDhfkwpIp?zpCYMGvwwt zMPQLdl#fqCu~{*R8Koz_gVw*l9Dnt6wNrMxb;$wRQgA?Q)MaM^tMHmsB$lF+`h%6i zPTy6+QQL0M9Df&_AFUp;r`?5Cc^SIEZCPo{3|^^<@$cnB@u)BeYKw?JIj`CGT#3Y( zEywze)3z`?zEZUfJ0ndqT?UUmu~H}3xG!_ZX^iN(s`go(aykjvWcWjr*6_apIVeTG zm&ln7BUFuWU9BSY*UFF0FLk2ro}E(xP5vuN;zp^e;9F-NqJiha+Q( z@Xz+#R3r)GX1RsFcZPoR-yR+wRt`Ps8yL7_icLC63R(v{XWHo;O z{wH|wFelhYCiU7IxSx)9fcHY>9guSA#|9qmQ9WeRvZBh_$=rt~AZ}l$yHNWAbu^Iv zSpH0o z3KzkHcWOL+Rl=b7t+!vB=2})#cjvJt-9%Rt;i!Cmes1EpFdti25Cl&{+xxeC`hF5@ zA_@XYOJ9Ejukau?rn~bvE&c6Vs|TEeiNYwaA)~hEPUDpypqOr_5W&I=(o9sL+eQ>t zh|yz%!-~d36|;8ql*=?JOg3we8AwXY#zX4=ZR3C=thY0-$=Rb*k0@uy?TB#Vf)i%7 z`9}%e3vl`G@RkAyZM0o({i`D?pUX#G3{7b9`hbT4iyQgksfb&6K=zE z(@>ch;217Fp)>q#E|8a&%M7v69PjAV$!e5PwYAPfTopvYG06mQz@HBY0?ymt?G0k( z!u5o-+Nn1!Jy)pCQ1$acO&}=5x^LR@J@^?^q-#$`hf}o(y>NRIecuF;)4g#&&4kRK zHz;;yZjVd3(Vl7vti@()Q3mQ$W;$zJZ(mdY5JDNv-h&#;cpkzwqvr;=%Jeq(dfE?s z$)MZA>tYe>0kYxgU`tC&y%0V4KpVRSJCKp^qL-ycMwS66-4X=PD?;{b-uDT7>hVw@ z^*u9G$5vfWozQ_pKPJWmHv&)LmW_3F=$Xtzr+Wjhn(%l8iwfCp{)+vJBk%7*a@?1% zM0;1p44MH_&Lzrn5=X^_Ssi|$SH)XD^o4EKs!C^9pRL9SD&L~&SwRu+oLa7h)X)dSiDRaiyN=a-5~y;mZXrsa0iA zLUZM)6jSg08p^2t$;+a}%l99l^=Sce-v+dtt!&cWU1DWB)7oUB0mun6!ZA+I3A8m{ zj4h?JI^~joZmcS-C{caL|XZ#{3M8aEk_kcRtQ zQSQC%7eT6-LrJ_>(!!lrqt>e|HshoWuR_vq#BKz4fKBl`NF;M1!%~;AVwc8HenF ze#`5o6u0~(m(9onU zXNJ$YnqYG%XQ#-Dp&BCACbQ+qt;UW9B;w$P5h>q)<`!V zh3F+r{Pyv|OuWxxXSab6!$FR1&GfyaTC##KGlWDhfhYtxrgIvUaQF<|zi$8UTf##F zN|T9ubv|sjBl{q3v3tEFUC#p3y!d@cUcByC(+~`_;(!T0AC=@y?|NphK7{-ja2l;g zM@Xx;&1V`#t$u-cJ@HToIs}D2J*oUEA3q?)!Cl`ioZTI;fWUI zmHq?}c%_3$%y3g2U(eF!N09TeK^}K1grMZ+L(IS)qyCkUsW$W`KPJ%b`5)|<3faXy z%G$)Bmui#C+=eK5X3!+gnC>zgVc3d(K!<%*XL%%-Ek&kRuT{j!=r8E=?eogcxE3@z z8{rdynD2&$Av$*24vc~b$ICtG>E-jG3q$>^d5O5A#k{lph>+Qj1FwvRa;I!gq3D}y zooCS**|@BVgGV4rl>&k`r|su|!@cW8CK|4}sqy*=h%(6x61Lf~yL|>_Ogbqnd6%4b zBdXTf08-*2M&F-`r&&=u&hV4n>r?5*=IriT`9Lqm3czH&*p_%n90~{df`ttkoAgK|Q?~?kzV*uz_ck>e7D|V>$N`;d2za5^r%orZ2m7wUwYq zy6a3DYtvz`z$a(V@U99hAbj=;ib8)~PdxJqbvuYj8+nNBRt_`z0Eu`w`ie$qIKInq z(~-dDM87v1g75kxts0PRILmbksXy+Zb)9kamUc|HmAra&_i?K66DlhYi5CrknNe=Z z<|Z3aeKU3neb`hpBe3}ry-KM?gRETP=!Yfrp#n6TK(nNM)tK<`mK&JZFK&E`= z@3{QzvntEZ-UzWD|7%58qIVx&CmG|>t@GnKfc_#p^PKpLk6&9K1x_jOtVqV?}@~?QMTI;Ua9ECs8w7ela6|={PR>tpW3HE=6i4&=gTxM z7_U&E{zU|J{P?IiPh=+4?yK9YQK(WmV!aH@ID2Ruf;u7V$Sc(KnkA7?3gS0Ph+d;j zumU0e)|`WGLu-97QF35cwUFqIlT5U-K#i zWbSw0gB!FVmwXgRB>M6&JQ8af!F&p%>ak1H49;=pe@jrrGZjp)FY+?;m60ZbOEPEY z&fotxQ*{X+E###yiekEh;v`R>sep~-bIb_DD$A#cf8-Rd%xUw<2f^ty4vUk zc&1uJ_kbvp&myvO)a(^#8N!WZ9gz`r>vF|mrEOX#L3v3E}wrZ$K2NRudK3RtDUe-9*{uGda z1Htg}_y3B0N1T}S2L%EAmD=09>z$f&aD}3;>2ycz=D@%{pS~`oTQ4xyn)$UQT8TpTW>R~IZwloG#S6pJlP4v z2rA@jTl06Z-2}>KbYfmdPr^FNt_B29kZGliGj*+fut}ACER^zAprZ034`jO-emg>s zCJdmCVGh@1?HTh) zf{6o}?I}BeQ4V@^LSeO!mSE(m6c?npZNXr3Qf$skv&;8|rXADZ@1(*E7%BP+orK7$ zj=)M%#%OOwls&EaUkiy7{{x$#%;ThPV&(lKSiJ6j1wWm<59v>2TKxBnYKM=C>(1Lw z#Y)vHNC7B?zX}7^=AKrW4AK&w^A7y^+m8S z0yrgnGj2^af}FN*SfOx2iYeMYUiT|Yg~$SVmb1NAKg+!yCu=P21v(5OT+P7Bjjpq zRGzBk0Dg)Zh9iKpuiq9j+_52ACdI#N-NOd9O!YcRMHTavNV$8FP>nQZ7% zz8Fc|Lf|=%#9LhB@T9hv ziUm+cS-AQzHY+~3k~v%bt{_STHAE}Rll999W>=+IQkQ!1r9_f##@%M{SE=G6_WkJP zF~nSrZ~q49H^N40&FWLH-A|n>O4UlG%;O=*^V)`~*_#t5By!6uEieOU6A-cXS?WT5 zUI^n0Ae^cLtRCKdpg}bZmZGV&6YMkpzQ86cNhbP~k^D4M_in%i?~pzaB*EnO2iio5fCdT^>^F0Bak}E5-G*f$BV()=k#p+jl2cS{&Qo;n9a-TRI@pJW{ z4UtF&%+Qp-RmH^B!26j70$1!7ovVt>jaLd~28qKF?NaG9Dg9=8eh}}G{}!~K!PfN* zcz^ZP4=%*J>~&I})V7+v11{LB90G+m*cEsh#mD|E%R`92cUd*EjMg+K|K} zDbZ!!7voV^QNmvh&T5#aeI%yzopST$lQm?s$V5AR33QYm4ewCJUOIrl{^ zgfN2xy&#DFd!{C_7iCYPh4)dB>J`I^RH!Bc(picw0Z$j7%VcYnQ0fqaid9hHTP5n0 zEM=pnH?}FQ;hM`p4dUV!J(#VgnlaYuFXba;`@4LJ|Dt^9O=bUk@5|5lXA~#|4)qKE zk_+bt4^u>)6}gx>Z*>}%6xz55_2o|QN_e_Ekq$;bOW2d)3oa}DJEjlaLE#UrejOnd zojMXV;+`?4K>(Mg?@HdEB4}K>V22lHujm7UqM~LA94D4DMBmsDt7tBm)UvrRDmye%qNe0%7ZS$QFl8b6?0Bjzt-pY7DhHBWoKI?%KHG_VWyU0G3)?1M_b@Nm8BRVkHKJX;kGnGP6 zjc2s>mjKa)_Rr~}3ORMyip>4PfExHjpvSP`KpwIGF0>%-IjP!E-KVb)^1QMGUz$I-DBY7>0Nn;!9jfZhvR5&@v69ZxJa?(cy1Kfwx>Q6g^sc)S8#*8I>g;uq_ucwNB);g6K8u!t zgfs1+jo<935ub$pcc7A^+_h})+>YJZ?P$R2HY0zVC${jR+r{NX;jtd>BmAI(T!i-H5%GP!<|X&2H282q*a~K;YZ(%m*?za_X_vi3!MHFL z>DVNH19j0eh1l1>uXK)>rVR;*3Z)`0z6TBu`{q1;3KpIlF9E&eyw^-BsF5NvU3xhn*CDdg|vRHafWyhfX2CA*i; z6l!uB@EBKaUp?~uA5sbB3k(aXqiwLexmf3YE64V_6Hfe?2(b-c%_Z{m(IG&!a6bi) z#LNkyU{hxZ&=JY&xm-#0$2zqzVW#~=?&1h)K_%)XSv&%Wn!eW!`4ETp1As)Qv_R$! ziIF%er@i+2*?z3dUo05AH@rT%H|u7sx}>>76&7t{rh1Syt@<}i36O(vY5x}Yw+r0j zim>e*Q)14hK)ez0=`PexlfEFs8tAK^2)$xP7K}#-W|?m1tUSd!elORlpj0@$E+_iv zgZM31J`{>gtIIwtCeMb?%uixQ@FGIUgtIYSj8$fz&1$6h?HFq#2!p+j#-^N$HB*12 zO|i;6=N=NPU}C;t!sEx7tYOWqTy9k5i}@Rj1E8xj2u6<7Ykn?swIzY1DRjRk8F-*~ z4@5}olm3D7a56J}0z*#P)$&N#JYNE+{y;#4y=WJr=cgy~FQTtu4<2mj9VyJ}U*8+q z>4fF1xIMhsxW6^(|M?ezt&R1b9{BT8G=?^v5V({!f3mIa=T|522kZYg^8SL~|9pX1 z=g%DI$tvculC8A~iR+)I8?mJS!vOx)b^hD+_^fkVjXQOOt1ZGUPvt*(Wu_RqmoV?Q z<^H`B|Lc0<$=(a2R(R-cav54Vn8EazuVGwB*mZDu;y-_2lc}1T35ask+shka6?J;a z{a_75UH4BG$^U+G!c3c6n*YZ+5MD8EI&nZZ7=NV^gl|>4N&X)nIlmN4U5$qSOw}?|oLr#7uYMH)(TwDTM6A(}fmW{Ef8L7Vwu}^VHH|4L z@RK-0O;|N9Q;m`%mFwmDOZk7c_J8i(#eS;R`O0VR|ABijb;voy;}Sl9`u5^=^lLV$ zmFt!T|H$b7MIqr3mu6@0Q-9$-P019=gN&Q?CS2chf7%T z_Z63rfD6pmk4spIOGpTTK=8S{!W~?kJRyE?J_ncwxCu&lz#LtjU7Z|%ml6^X65jNkSJUB;+Ncp)Dz;x0E(+M!qFm^Bz6Lqv#HZ^on@OSZ1_eGc)>w9{L`1-0K&7DQ< z9lag&wB7EA2I?y4J0VSAz6d3Rx{!dUE8I&LqN8R6Mj-DeVJPe^dV~dK>BZ8jB!(UA+-vu67E3X5cM9V{L@0YoM>Ugh7C+kD<34SfL8u zCc1$_2o)u9Mcn`&gomlN7(~(CP1*mBgp#ob)DBecBCMe;?1zvCmr4er21p4L15E)n zBbb_T-?J)$P58d0Yyzslm%2xjRG7U)x|^=j5Smpe3bRX zolRZs;qIDn1x+2On!2fgsHrwoTi3+NJW$P9Pz$N6E2i!!VlQUrY%bykcT^Cw)7294 z^H6uwl@t@za`964Qn{m~sUaw!>8q({>a3^XX&~tZF){GhyCdKyq%UeO#Hc6L-d4|U;(heKvNF`V|fn;Z@8hJKf+%^*+|z-Qpi-@Sx-<&+#R$f z4^?vrbn#b!C<%%NsKSj+b)X)Oq7L5Xf#w1bZ)I_#052a0M=fUuI4salK*PvUK)~2d z3#R5Hu8wf>hpV`In|iwGh#|n9SFp3!LU_opt?`B-8|8cg(dw z0WT9LZ@8BOtz8v6BPSOJn5UDSoj{n)=5*doJ!#SCTgju%5>qZZI;~ z`Lx8Lr`~h>{tZprSZ>2&1R>3NjgCMnQ*dnbrI1}!EJPtT8RFIHi_ceYl#Z@s&}hwjkD zu>brK)=Bs2r0K{_ZSVhDbPt_MSab5xy?ghaxV`5;eq7|C+lN)R%hUa@CHL&5gO>cq zx_-A5o_*+qDR)Gg!pZ;AG$?)uU3=ia^$-+`yL{U=?Tr5s<^Mbu+S7|=Q!M|z-4kk$ z_JqUrzw(^V>BuxIm)F}X zskl9EF}v(jc0sew>ES%rc2}|8w(EA`krSqQZr#~ljfc<%Z+!Dvc0E6#1QbC#{qDVR zaRAr?gJRVSx*T-SgMO!sly;4E3z!sNvws=L>|~(B3rP>?=M-%h!n9vK+HDrYMF)5F z_hMKWJ!pjOi+X{vGQK%belW$L_)-7CUmXd5uz$EKm~T*U6o2{Zk3F=d7=Gr=&qj>k zJE{HB53gU^Wt(03fIXUGx-HVlT^6kp05<0OAQ1)p!_PU#ChA6EsEvSzxw=@ z=eDg!UbRNs-qnR5i6DXkvXT1m5K7J8QUa*PFLN z+u@>#D(}J5lh3?=O~7AbH#|uzdeMF_9pOQ%#IE^0al6?WxECpJak>}6t5^eL#b{%w z-QTnkT=RrDWZm?&miLO*GwBhHPj(JhLf_uKM27bs--p$nz@w|Esu?4GOiQZ6GiLw>Y|fxM(FUetO?FmbB4q117*_pdf2PEzuNYC&Htqb+fCH5e z3o;uaQF#Z$s~S;E8m?gB%;apprqB(&;A0+3(yshCz*nzN(R|^gVo||m0<=x-cKo6z z*f$6=f3f1O4K)f@;=+CF%36kf*tc&($9C>%a`YOAH7q?EdTpGQ+uJ}z=~wrD&|i;# z%4V{W(_@_W5It^)r1Y%PZ7!_Gg!sACHU{b@A}A<#K7Oz}^eL z)KZ~NSUJncen*xB)Ck9|w`hhhND$on?g}XJB^D@j_bW zo-g;*iI#6>Y5B~2)A{(Z0=v5LoG;;VAyx#c%_QBJEgrD8f9q$OxKqGLN=FQTW8p(T z4q>B5X(1|V5G!KfPR$PYhdb$)xTpAS*T`v2l`T&-U3)U2Ukwu3C4)_!V`lh^(owxc z^E7Gkl0^9$aZ#rr4dJ*{Jr3+fhr!%*V@ihgPLdBd^g>KtT)>9UpuoE=vI`Zh8P!wH9b7i zc~|z@VB*p0DfIJ!ZaG)%mBHARIq#5&>&|}GEB;p5wwcf90|T%^q0>1|V9-(J(3MSR zd|Xb~^T&lY*9TZd;_3#iIaZbTLv3&akO!kZ)R2vduX@O!jlSh~4M|hMHSh&)QYmzH zA%z1&{rHD4u{c=%PO7(YI^N<&>StY2_pRt8(n!rz182RM$J0`|&G*4$G6$xO(DrgS zNGa71)}5kSB14|D5i#7v#_~}j9Bw?jGWo7BQEg)`v8|TL`03XEqf@io3wX(_1MNjR zosVads~1OqbglJze?5y$-CEzG>!xp0s+aw?Zk`BcXS+yND3dHZCB0ScN$n*6_+H@9N%V9XQ66SsVO|o= z!!KmOc0M<}toCS;o(ftd`Vu6D~x zaSc`$V3$SCH#Uy!!HlQE8dooUFaMO+enY2=)U@wJ;o@6snFl5=Qx=i88U1dJG%2&Q zN`8^~Osa`U{%A%2WFD=F3?;R_?8F`%VQ!54EQ+XHVohE@faR6;ON49->!qhl_5@io z>>TVey(Y>c76&5-^BElQ>okH<8?A4_OFR?u_?C_vRfTazhCTx?w8{i9LIQ)S?r>y4 zoMFD6bU&B9b>U|umCH;bBwPH;5xDNlZeR4%Nrf`lh2{`C&amBg zZX24mI$=Hey^<*$?)6#e0G4H|u{t39X0pij`5@&h{Eh*1x2fzG1KwZ;3mYk`tz&8P9MKmBQ06+K8clVt50txtvzU{^zu)~i$igG z!;H*a8%KYzyQV$>W+}Eg{gE|xUS(Fwt^AB!mCG0CRTc~2gUa1U`bUeFZM%;|3PA%x zh2M#$YT2M?k|oU^r5g*S;wH)&!6biF+K;8swGE%`ev#NH!x`T)`^AeMN^s|7Ilp#Z z8kZU}{Bm(Dk$aI@YPiGpRG0tHPbeW@7g#HDV$#cBd~;u&i76p>b9SD&IPiG<#LDE+ zKu^&iyG6|;>A?J)HlbO=4W6?JP zI}>E`{61d?D<3D6_t`f_`}hXg?M*mbA{6h!1)KkBWlj#dI5SWGkm5-uBOXTcF_L8E zZ5MWyFP9zPc06Qy?WvVDfSjDlt6KgDLWPXV56AR2`9?%oP zP$`dtYo?D5k;V@K`*WbShr{y zsV8fBtZgz*)-@iH?Vf+|?l${Oky%B(^qod%ibhYLMlz2X{79=J=0g0==6B(t?d_mf zoNdy_n7JsngZOh$mvb|L%P8Hk<#f+-Ux( z+PkM%`cXP^lZL=A%}Z*c-n~ActO!N9z8b=j+d9(%A8Ip{a4uA7hbufYy``~jDVw|5 zG;X8|Dk*1ksf!E1CG1NBZ<;98*f@rB&mv3Cng~MhT-IC z=Jof>Lw$)RcaZAbkf(nEcbTgr_`^$C`;X(WJK-$MwM0T;GdDi&iM8eEIKaP z#M&X({CJg_s-VR^SB7AY?hB`S8jTRuUD>ZH97cz8Vl2nDjEN{Mau2Zvcnz_ow>IC8 zj;yi|E`6X z-W!xRcGSjWi0ow5uTl3Ej#`%l&S``WkJ}=V zQZr#vk7)hwI^olKP0)7E($$OFnaMX~X>ix1&`77vxX?K_n!Ls9JyzuBp<%cZrv0vz#IU6n;V zo=4EdS(&0|TyZ<$4+fpr;DcV;gKVpsmx>vgSaCjy$S`BdyXC!u&d0H#*edZBX50lg zD-9hpY~)aS#{MHisBW_yZTi_nQPoEo-p=WTgr@G!^zUr0CDJqI4?GS=AFEBTIS|Ei zh(K=>YH>4DO<>}7Ryb_7_Y<0mUc`0^Q(A5IptD9u;ly&uo#^j<4)xJ(arZl6q|NVl z<$?$H7}G;=Te26|7H%+z21cdW<(3Z|oH)wPY^O4U6hn=_TxgE$Q1gI}h?Q4QJ_R$ahvZfsqi$iE95WGXVYYbT%$u(^WW(ViZaeI!7f6|( z7KgW*j=_SS9d;K@?Q5=np9E30VQ!Db#(t)tGTW~P(PSg`hd;r*g{rRlkZ z|LTJdR@n~e-l}%R(>b$NY_CFiw7NrQrtj6N1Z@Vnw$`wh@}u50h~!vAUP^NId((t= zz@`3PiNvFOrC1s}l+RMFZ349dtbg{ht?n5>C>=xG!qu5|6?u85GwY$HEp|fs zG9v2@pTPg9nhvJa$WO=7tPjejj!lBXt-(;iEBr<)4-p*x@tfF_m1!!Pht{9=Q^G{P z9=e2&9LmgX9{u_8M%>Vbwn3+DmS-@>7wbTy)3b8UZlBAfQoA+5qA6S1Jc@?<bwD&Q5>ytc;kPK8U!Q7Ht9P{X8QqGV>q=U+|JX;=?$$yCOq0ti*rl zc`xMM%^sq7Xo~2!bL^-H8q1_q;^Lix zPh>faz86Y3VX8_g?%4j<NhncX{-T>1MQT?d%wPZ@FSk7~{j11v!Rd_}ouP zM#e7DfeO;PS__nrpP_pxjr0~#CR&Q(a3&|M=UetHuZ1x>qw;dt1>ouUf}Y~H)z!!^ z9o(8~95KA3qK2sUBcg7DWcgb7A3oWnYVD(?jHZ#@L07+>nNIg5_+|GUAU`Ern|X$0 zbN0foJcpbq92mn_{xoL7yT+%TGmQ*Hen)Uu&ZSLLT81|WUrG0Yg<*yj76!&p&S`Tgy$9@+3!-u=?u~^@cb}oQwucwiQ}CK+fQbj zZ8k5Vvi|eE&ui&?YW8DU&KFxPJ?XfB{Um+aYE!ZF^$pJkMHcJOid5fmWxX4 z%sF>dFIZa;Q^Mc5;JX9M5*uWmZ&LcKgq&=}iOKU6UxpGA?4T8$(qKGx^8Do?v_T@l zd)r#GH^V~p7y*&4?Hw}yL3HcfWMxqB8+q3&C0M4(Qd1-i$7~vVOVhX; zN|2LPdNJvgX(DEF6SkJzQvu_(u*Z@}xYf75k(pVYigULObUg7t)6FdO6kXHI5#Lt# z>(^OESmv3N=Z-LskUUo^bPb+g{ccxH3~ZBPdlNkx&9z=5CNMxxFdc@m=}k&^7WL5+ z{=z6&@qt!N6u-parKM^agQ*~jWNMf`eaYZC{-?^%R5oN|#2cE%6Y{aqG1K>!{n+h_ z4VWM#!n{?6Y%;5%#2^qp2|*Kn-0OW<^TGqKySv)r&aUk6(%T*Siwp3KpnOIrN@g5> z&sn^6Hm8dL`pB6O{~_FZc^SogvpF$IY|bUfRgT5_l| zcMXJjv~X}k`eMKSRp9eSo{m=uj4-!(`FPb7cPq8bRB=|PI{0E;<)_6MvDO-!kgHO~ z)gS^mi5>61WIFQycTSF?&uVgrx!D%?vCbK$Y*)Wie%uO2z@JVn5T6-5&Nx?9alpeS zl=85l$uF~^)?K}%jJKas(gYQxXzwq~{5F4*{zvCD(?KhTVS^1NnT}|G5xqArU*EqR z-gbC;uZINuNemKs6LJV{4jUd69Szf0y%MrRVn@1;yPNIIJqp_^pJPtS zHi`+?uNs@Qi}~^dh#&Z_{i5ef314j_dxF=tiWxWx8G5W4W77xm+W^|Wq2h~LPsrs* zpqhM(HZ-RjmVBCt$55{cYm4;R?m30>_Xtc=UGblUBVX97@C8^Gioa=t(HHXrVfwTL z(lt)ck=N#?)7wOchwEaRWcnI#C?Ag^|4XV?JAY!6dK$Rk7GLV-ctNYfkJ+TtoxNOj z?7?Lu+%J7ZHmT&VMN$MzP0{?)Xp4=zH`CW1Bw^)JI{|pfhF|a>lWv)n&Le`P(<*!R zZhH{639xe<{*yN>la(HNBQBa{sKNhv!^(Hxs}0-K8pYaU@WO6(hrA7|dP#kEwo6sP zi^q`LuU*F!?-i>SNSkq>H?_T47V8qy-hkORww^pP@*rB`rhes%RKqt94yOb!v@oEHsWZrvKhhG@6A?CM3EVR4h_s>2 z9OcLoJFTlbhqC(;-5xdQntBF5c0Vz@@Cr^lUX&^p7$FQ7yPc`w90lJV2&KvYo4;T zO)^;;PuxvOj|xgLOn5%1f0iH1_JcFB#g1PnrmDJ|_ouNZC&r)bL;SYcJ$yYU=U#1W zG{*?RRs~k>zVwu$4;#a~iM+Y_@<@f$}&#xSLBhEjGu9p z_p7Y)gd<4_2penX2gJplmz529DXA|ox6?hNRBb(E3m`|fpAW^fw{cG+ZKNJL{&1Ax zi2t-=nJIT`sUaHG{%EQKu}a2IzK%TDM$kBy+-C(hxG^i^bJvV0epZ~0FcZWDW?-~e zhCf?o=c^&?cK)XW=85K^5LEb$Q6HI2qx$(fv%+=>%pWdv=Hf#oOEFiv<)kDB+l4a| zX1|mN=Uu}TLvPhqkZmQm*Mnbiew>bGcayt?y4sl_Wg>pmTo$03rSL0w4vvN(G^v!yPa{s2HDa$S9_$NpRXFMmwe|Rfs81tzz2G|(c|p^5;c4`%Y1NTczhmfuHp&;_v8?;>g?Rw_{z1JRgdYrY(#LDRV$6cG0^f9=>X>)f zx5aEwsE4rvv*Q+{IfIHpXz8F{jzC9`oi_eZ$94V8!|8iI_(-(4|2$vurtCL_-|J6` zc>urtmd-PSx@VNPAB)vn%e20!W@uYn-IUe*F3nRWc2M!=I3mAWM~1LqVO*IZi;pcy zKbf_$oOYs6vU&%=6@kkxr7>z6{RJ<$*A}l?jG@{u%n>855oE8W7F3XAD7ej;pW9<6 zN0*Y-KS;e3A1z2jLGr)E4zc(ISsNZDbCm%Q?_nKHbQ+1)t#kMfL5-G{HJQw5*5990 zTy9)#*tD>@b95ljlL38}vj@Mh536VMDunS$S#&8kWOrum!~+aimHwXF8n^l9)eTs;ABCu&x(07~S)EWC7@*|zkkVBI%O;Y}jjj1(X z?f2v`&O5~s3mcAk$V1fDNRAgAk-|>-(e6u{427TKuUlvz2jI`@TlY7N)1)^_vK`nS z4-oBN*3k_3JYtwOw>36C@uS1H`{)Mq2BG`bkM&^gb*ozAwY8}r!_X^=~s1AU;$?}KD$jTk^`(iHL3K3tOd7-Q)Ma_cfv$m2l!?*E|@ok z3{pd8On6LZ94)YoSIdr*S`Te)f_ZaLjroj)L$lo{-JDSnt3kQ1)x6VINkT6llQ->u z+?~z|yjSGIW}g;uB`&rXJ3-fZ;AO`2y<6KCC!v=%0wW|WFRyzc8hgnSF- zU=Q6|io@8-6PvJR-gkLLGo3=~;1wP0NDP_%%-nuHolCJ@4X4qUV(aeLhrYs$EHohQ zH=M96qX$V#so1N#{a-#1X80oya=$`)9!ASDz6)7jV=aIioi4yv=FniYRP*m5M8z3e z;-S<(^d`UTl4%KCm^c%HshpD(&Y>)?nZjW(Yt zCN@u*rn=)44~7a^<-E1dco>rmKd0ysgb2MkQCA~s5EfMJHCVMepOGIq6u5ow*5b1y z#h{=(g@KH-ATeoPNw^G8seD>-Cxb}(LaG7z->l$jkH-x)KD9|I_0(^Awd^N?<}P?# zJJlvUdayFHYht4jrd#OY7HAUWn(Q4T?lR!oj0i5{>t>l{<2ZjqymBG+#v5;Q+XZRf z;9?c6$!K=6#!#D)vMSbX1mW2hdv*oVIwe}<^LnnGp{{EQq<}a>cSw0Zq{rV|V#t>C z6l4!i5XslyIH}x6fBf=CO&bBp^%l9IO6TV@$u$(SLEgC8d{LKxz^Q$$>G7`7q*c6; zIG!t~3)8&Gmgz(VFzhS-<`+`yW+_8mk&8=B(ZxSs2d@Pys>qf`Yq}@&=v!d2zv_-B&uoPr*aU=l$b&>D;cbeu(Zqz zPxW7uoKoJ8TV;Q}Qx8*Ct(GHyH6+UR*#s2rotQsKbsDJ9xu04xRCBh8Gm^@C>DvY$ zmP_7E-1BOpn47(|P`PyrCGF?X8gIG%Sm+Pxz>UTaNRUJtI#P1q;?hIP z_A;6LplN1swQEg(I+?Jtg~}!U+>%`;K{8%o+WkVib2XG@f?+pS;$C`A7Fu6sMn$%M zfU~{`x@X;&;kl`6Z6>0MMF(6pA1jAjIYv*mQ@SNp zPnk;RmFDy8PUHMZ_Ytfd@dzg|N!KW6>rKx&_lo+KGasI+-_WL>Z+%FzbE)az(XaKV zmL2bVIsd1R+_)K8Uy?H&rX<&a74k!U0{NN;+6cDRbME`}D>K#dmrpOf9L3)o)g^t2 zZKxR=M(I8d)0Kq?hUzo%Qik-I}E>r`ZNlNQmCT^aB@R0E4jc2XCyEE(jdonJLz)-2 zu*IHVtUBR-3%E;Xvl``IHNxh|_YZ`EJZVBvZ+(q>d$-v>Ox;}I76AG z0)vMp`X--qPY0@gh;VbHu?PUhbtN=>N9*mZC8x6DBrP zdQZ>;OJ7x1Oy3o0-ZHG%{nIC0F8U#M4(2!(^oG#y3GKT_NG{WHcDm0NJU1iGo8{Kn zuBd&xSl_w8&E3Z_iO=34G_ftb8 z8(8+?9|KGN!3C2-YT9_$u=BYBSn4lMQ#^b5-$h1eE1nKuHJ z^>`Ak)F)av*tCvzIg!mTFjG}Z9WC4F)NdNj89Oo5Fe(5{U8DQM3)>4%lL~j{M$9!P z%}Fc!I^8I^bpDQMe*y&ocuPTzB`iW|Hna9T76r-uP@j92h6}y zERE$yc($L7CsbL?SH|Viwn-n1V|e3>Z0w+SgIo}WntwF~TavIKN&IY*f%=Ymt9tA7 zbV88}_8^w&SXZg3==Z1m{Seu&#KY*U!N(-Zmv4dHC2ZwXDtxN$OA^ECSV7ADrfHLW z`*+j8wVt*zZ?37vx>jl1k60vDHV?1Vz+`?o8!Du|@UI{f zaGW4e_;`0@l}U@N`~%Ti+BwZq*lLN50F-SBr~ed)>XJ{k)+K0rio;bw;^`C@U?5&I z>tc;+{_P_nQWXEp4+9Y8LQiT6Agf+%1jN6-B8)5VF6;132~f$n*UaMxfROg%_&RI} z8_OMA0!?D!eR+-+F1Q^A;ev5!wy%fZnV+kdWSwYH+Uqz-+ZVGTXE#?d8oQGPQ1V>* zB@XOng*I!0s->%}^v{LEm!+3KkGGWn$souTqP4JfPM4ncHi^apF4gIMts*tBS}xh= z1OQOY6kjkpF`>@qM)7LdQ>_ahR z4Kpc+yA2y#_AB)(H;iNV8cZj_PY?Z~>AeCjT=ZBprvuX$5{?|YQ{1ri-eD@D5+5q< zsA9tSLTC^j*+Dy*qjUo>UyCn%+RXw!^c4u(BeC*v+oiJV8sTDt(9MM4-qwg5LjFzH zuUp2Fi*F|8i_hoK-r}R7rDl;*FC{VTG8a#8j^0QaL@!&6h>>Fi2qx3*h`aRWif2#W zzGwshB!enQKGUwL2@6xDou0V#n>%YSby3&W>Wzz_D-Q9(a-@)#vE1R*lr0e8Hya?C zl7032=>Ta#!Fm!^O`^hOuhorttM&vug(6T!6BqZJoijstwFZRbV{ ziz$JA{&C?z{2T+T`k!d0lVepQPTuv<#c!YwP$q=uBe72LGCrHE1jC}x9FaHU&O9hP zq~zlIghAz$;nBVOQfZ0L?6FrRyF<=E8aEt*`PA=5M|iO=asC%2SeZ8HN{9S&v_aq2 z1Y8rrC2xU$Nq_;|7Q0wqe25NOL%O!`i#e!E8!m0-?Aa?0Dz1NR;(B`5Yv1@l2?QyR zw;0S(M3TaV5{4E%JK#Yps%WuWVRi-_do`(Y+3#+GrdpJzx zanNZs6R^n5SV$)N>(G@Lp2M%get;?+r3|(VSQFex-)hn8BOzvq!Gq z-Sy_h6Su(>YjFOP9^JebjQRM?mBI(Kq9I?ExLJ_p;3+X z*BF99S8K2Sx*P?x?S+rlIa<3WB8*1<#SLUsSKeef=HrFqQOK7WFOF)_6aJzl(`if9mWcosjU_&} zA~j@9@*z)r#pQ3bz1x1-jP79QD7SFrT0*M0H4TM(R+98AXU3Jwqv$xTe;j~@dhSV( zuC=v@Dqmqp!77WmVpnZh@b@D>Iu8Be9lGhN5S04&0{wlZ#zR|7bH^cuU>cdJKN8Hh zSIGGC#euWsF1^<_rdDvdW>qYr&k5#u&a?aloHX8GSwY}D89)D=0oHK&Piy^*h7qjs z|9|Xe#ki&0x31{*OJAU|WNLX-t<6^7G++XABz0N=Aq7X}h;J{!s4q`SUz+pvy+}6> zwqfLr4A0ikMFU|dnun*|J zqJORP6FJQspQ76yZB#hUt}Z!Cbpk@A;9lo}z{JR7FKC%~_U;RpY5BYIRT?D{z)ctP z(u}UpH0{G);&B+0SA0S9&T6wFTf3qNog0huqH3Ot{Ug{}ZwJpq++NvW+i=!*_B`L7 z1{5I@*HIdy%-R28NmhBb-HOHJn~CV7U-=ecUGvL-61O`Z`vP3cb9?a%&%2=$Wr|>b z9CvE_du#lJAB}%*P=3E{A@WHaEdyh$g&y~j zIN0yWKUjBbg20+8H=&AUsSKdL?)?KnI#dh4CcH%K--I9T#=C{5uV ztKZ+W?UzV{-vBo#rM7z(m9IN~S-$8kwR6$>kq(lA=&YzHzXGU6NJ+bIPaBu#sO4@aph0VZHnj+}9s81ed&HnUtR)4+!)uqi;wi6GGfS8TtqiIghKQ@J0B^b@C z2&;e)V5I!r9*L@}rfM;N)soxzNne{!$?1iTK6y$&QpDu*==gQ>pT-vlcsI|sh#nhs zZGlS}<3t1kzPU~8!04g}72ZJWhix?O_yIvxSp@bnFMMru%5wB?eVVa` z;s(gb<%I)L$Dof~7GbrUgX-Z`O;5?+@2Wa=wcOL>JkS}eFemD|_~dAUxs*9rFg-Zc z$=wv(N*V?Cv1c@l6VN{htwTOXO;uB8VRcU0vCu`A(|)qurpJoI!_)J~f0#L&8;7!j z>FVuagIeK@x|{1VdL(@e1Aq=2mwat9PX|Dk#GC0-+6kJW?==}^1B|PIc~+k1u1*c# zw6K(%@GmD~h8=Rl2NRdiJF$ryYoAe|S)G3$;u!!s+@LX1zYH8&;rYtLR~Sdy8&uU^ z5-)e!IVraL;qdX`oixq)KE3M628Ty)in>qKW^GbK9s_M@$iMGaX|!C&9A3G)VbfTt zZ|Z7Zb^M$oSh(Ee^olu8RhL#OuW0FEl-VWe8wTkc-B(u1-~B{(XK$TiE8tNtz6=a< zw9;eLE-8UCz`(mpI|h$TRJUFDd0caR&O$ZSRaE*|Yy3I3x~uMfRS8Qg;T%r(#OSNX z1{M~#8QVZ!i&S*xTG=+pLOTs6F#d?j(Rr~lzd3aSL@$MwfcgUHsr5M^o~EY5a78y9 z-3RMCiGFh{+s`->g}Zo9`!jkVi5|>-TrKDok2LKyU|578{kBL_0n#p`9r5WZqB=3U z<>Nlf?vV;i-U8S)+|pyAI<(^f8%E%4OtwOlxexO3NVd-c#ZJqYHNRY{jS^8S?6f$O z>CA-6d}-7j?i1V;tXC65fAS|c(0y>b#4u8bPxqIeDQhNT)MiR(kKr2(YP~B(s9`}X z_bz~g+B~c_xmRIn0X7z?-AFL;T?ApbC%g7^b_&=8b)%VCPDAo;pPDa_(f=Nzb%YPZ-u2TI)sHxYjm?%6**aeLnu@Hc&}zFzt{&GE_cEhlHkV+jhC+U)Rn z5>P8AtVPJ;JK5v=un(Isvl}F*+8FUk8uQ^!j9_23Zmhxg5C$)WeKItwee|Dt47efx zvfAG2&;30u`8h}MZYsx2{s)V{;wk0j6oi^RErjx7Bh+gM)Si}fx--q~ z`g{1tK~?^`keoY4htUQ(+@mIpY3~B3%H(qG21k2T7E!jTXC!8`{}5fBrr!GeXXN%m z$dfRvvp6p+(XHm9vmYH^b-$g-+Op*dn$LuEN>~~GT#7-RU-Qhxv~=WUl8##A)y{j>LsIa%`{N+Zc#h$Zdmi}UCX0_` zcYd4sQD=WvkWcG}I?uVRqC?Ue?W&?*HqDNuf4l@v^S$c(V`sHR4(U5&j|}{{Z-wrz zrxcC?xV6fn&ud8IP3brE3}&v*+fTfY2FlW3NnEli4jbE@`L54}wVhh*Y3S!ahWBG2 za9@>ioNLuE)q^t~R0c!m7a&Vn7Y8?k(}Gi}q8= z$kdBS8LwO_#canr`R2C21DGL>et#R**Nh(8-A$uY?!y7{qqr3Ux$p?0xrcbz9pvDg zv6XAM?mD>@r%uoKlB}y5;|7lpxPKhOakRf#^q3^*AOIW#dkQlD2XQt!XG)tXAS>e3 z^FA^v&2!LcloQZ5JNiPXX)H4{uK@;`gmPe3&!0Azfu{mxj!o4V4DtxtIn={5G<4zt zI6Me%xFV?|u$MsjfFLnnM)+ZFmDrBrUj$apbma~h zvR-*A(=JPS`=T}2LrZa*St}qS(}RqBHN?B3PcNOo_g%mXm<=i{iiRW@7WkTdIk$NL zi&$g1B955zP&vCf5%^p9UrZ@9Tyu-dhtr-1KU4;Zo3bYmj3qk&37* zNn>+oF~Iw%0-_(Hc;V^uwIRFKRn;G@=$@&(hb+&}yQY1`|32K>ZRin2vOcwl2zcW$ zzI~2*?6N-|@8|rSb`;Ah@+R+UOl%%RTV%Q}(}T%X+$H;0df%6JrVV5eX~r!c}o)ir2J&l!d=7qD=M_Fs+^;#=J^b&(Tot?8}j*Dt4;IHQ7`Y>+Y;*Lw4^PZ z+!ydqzvp)INg$c%ncV)uE8%cAQly33>26!$rw`>i26-eP3`G9ORBUtQ$H}N18?1wVw#(8>D_hyAtw%iiIb)r(t%0>=eRj`~|OjRPkzY2#b_r$hyBfl{8!2(B1<&CG!QVVsd2X^bWXc}~4naG8C zw-4J&q!-8D9z=%t1%x73Ha_Gcm)PwVHU(^29N5gPGF2AJB(n}{0ul@=Piat8%Yt;_ z1(X3sI5WXygfXP(gjjY-)M%QLVpMI5CjtNKCP%{t_#01@5gzMsQBHK$BHi(0;2?G9 zq+R6g272fstllZO`zts>Jckl6pttVV{h!EROoy8KhH7ZiteeN+6#3U7O%0dPKKu3C z-Y%w~k9LBucOy4pw@(2l^w|Sq|Et63v|--@8=^@@GKil^f@d42qQ!iO^b^_5>f9d; zmjH8w%iw_jGab3DX0#a&q{RyKr_1xZn6+$+_imj&%`P%sxowhEX)pHjI4~VwN3e#_@Du; z;_Q{rhkXFX*HWTdDNRX0(LA-iw^-?xXb4Z<)#)QwUTWBAF5*9I)d4H%quf4{_3r!j z8h?rb5%$-?QjZ5tYq9x!?8om&gW$cA;KQ=peUs1z+RWQ3Ve80gQ*I37s5w0|Ch_Omemj_5x4H$0R**^0$~x$Eh!4aVsC&r!IhlK`$;^s@_zsP-ZdRENRF6c@(qN zrp&had+0DZ&5=aBOruWBXBi}l*G+08-ImWI1E-XDjF&Z;rEXZX85J?>ou)Bhz>#}- zxskO@Nl(CM1xHtL54y&s+lKI!lPA*z?xr#R!}*V9`O~F2)YV*a=nsIb9rt+8{`#nU zz{J_s`QSWBy-Im5a?K9u9*_VQ!xO0U!pWM>LGhvsfF~23=*W^5 z*(?<$H8PcJuBgsdB^^8_d@`~y{W>j}UY#vTQDWy6YeI$F!wJ9!#>Dvktmf>4GotY|tyZlEhJL!3q9AflzqZeLOwn zW;_c6t8!qn<)#Halp1lrm3)lVA(w@W3s%LQ6Ti!?a=pT8`VEGh=-ov+03-b z0mqOz%~LF6{yMP6vl6{X*7!f9ePvvf+ZQe%2&j~x96%67P+CExOC^*J=@jYijN zj8$+R7I6mXJ#cqS-?jQWmhUWPZ?Dg>T7bd?m+r)&prs5^q85Qw=cT}WXBGxix<)3B zI)P5Z$G%T18{ZiIU0QH0QDDtFU)-N;E?~-6g?Bcr3<601noZtzPb&pf&uDModd1S! zaX+k}`2;xPYE)o_mUr1d*$Qpg$#{1htNThbbc?`!1G;x{5VLrnIvt-S0~CX}ad3cu z#+;wGXsgb)`D_wh=z11i81rDq?NIB)@?-rne#)H{nR!d=p z$?mndZ!6;pgk!2Oo2i zO$|G!s}}+rpc4@OI)C>bY2^S^P{3IH`}+PYwqMkf*i> zM^t0leb4rP4T)LyAXV1>onwyNzu>qF3Zox^G_}AS7e&zn&`w6Fg=4zthp)={k4?=B zcv&EvY02F~sZlC1y?BwkN*wBMfu+a=d{ihvrMR%rKzwo0PuHCdIJMQ7e(fiVgfMQ1 zf}PaNndiO(%XQqk_e3cn)}TAXPEZFMuJycgKM33y(Er2etxL42s)*86E>qQVxgk3T zUn3BEV7K~pGkapWa)r1qH$UGObYvjtx;s)fx2T$5xb;EbqH)gFy?r}+Ys%hreS%hT zZ&T#6^$zid#blmJY+OCa8CQzaQU@Az1`Fm7Pz$`fjN?)U+9|9I<=EsJHQNi7ZO-B^ z?ZFdbTbk+RrRus1WK+6|RiF%P%*ap?c0nlo)^~71%&U%BhfVzbWk}dt8>w4`hg&Sb zd1a_Xr#k@i0gZ-gIvc!MVb=P7XwQ4+GlH&dXt#BH(PlX-d&w!^T z4?Un(0OcD5vjl{4kAy^~b4JczCDK%~g)Hq{g)el@SEZ+~L~L{ah`&yNJ#at;5@{Ii z#iw)hitkCk*h3o?sxkB+{BSS7(Dh{qwn}%E&XIN5MYS>Qy%nO$G?wT*T#IE~EL_Y_ z(f+jXg;iO(qPk!^;ipZ}P-C?^V31nDfZkr7*gl2JPXGbcG#_49<0~X;Cf-c0Z11ysmrn#k_qt zEQN6VBS^YEcv{L!w~+LQGDMV6+258I-MaPx1Vg*sBTyWCl)=HnB@4MCQLKojHP+I+ zG>S1ssCt@|g*`2KWUhpPx*j5Q^Rx0wh%P*bL4X2Knb0RzsxPAqJfP!mh@DZ z|4$&xdRS8$oG+6vj~!Q}%(#fpG4|$o zuNY!NP!Sh8&?=V!PKJx~Ze4NDd2Uej$77sIyEjG9n#rhmi~02`6Z2y$(m+bAuwj~N zQrqxs;)^kP(+;lLpP|%06owW>j%u|{8fc5hsaGkOp^=!Y0%Nt=ljJ<%i6J3Fr;?t( z0m_hqYSfMG+?CCZXe0YI`rh>}LyPs6%r@g?0wMG+kp$lB5;|V1v~h4Ho5pZD0SQ_n zp)VbrfoHpzUoSJoo-1y~bRC?E!YzKZ)6@U$rQGb|=L z-DaCrJ9Dnw1$$ej;H=lpUU1S%F`I*QeonmHWl+xbSkm+`VKD$gj&o%^Vs;)FY53xz z$oe;Mr(k8`jzgMy3E|E|X!>#dGI+vJXC;fB_QK`;DnpSb62@XGkkMIPxMb{n?Z>`$ zR{4GZK8LGXM2`(rL_lmUcp@<0CIa+e=_V&bI%Sskd5W9-6&MOvr{JNDmkc_X+P}zU zJW*dHqD+${z#~Q26EusG9i_Kig-x^etfVeH3##=Sk0gwQpKC? zL0w?#Uq!DWhIS8HhYS1tLGyq>9lnEj8}a83_~&Z3;0*VR3W;xq^Or#sM#TL;n)-m}9%Qdapk|_nZQkA} z0B}5I!afA%pB38AMzazKL9LA9un@zC=#>gS;^oQBJdnZv|6XY`BnpYCXHL(b>R0>A zRq<4&l~YYf%?iOr)N``oHG<^FOPa}(jq5Hl0bOaLwM+qV->Wy1~@RuUE-ekFv}G;(A3 zlUDU&^s{Hrgtu1b;>|~^+)mZB+>q4hOnEXftkc`uYyY@5inn|tKV&ykJOUg}MoT=2 z36SjP03Phr0NYGCipOPozm26y4<7R0gO7t5=A3U!Cx{X?oC5pLTUolzWD6eO9oOa? zEp?n>1**k1ZAI2_wxGJ?Q=i3s1Ej#(iMxH#=p&imU4sr`4y!40j+s!b6OzD3Zi7rJ z#%(*VqFCx+4NinCn$4DlpNi2`@BqMVrKzf4Y;P$n^Ux70ApnO5G_# zpR|M`A-XrG>X*z1L(z;396UWm;W9AA#jT1!mYy=_V%CB#blNiI`XKtoZv;DxB*zB{ z3S%^2%D6;*Nmv@l+bQHL4vR zoAu`N(9={EgdI4{^TIdEQMT3=*1Yqu?^#EP<45wkx(u8cHZ12do6YP8+sy?H5&G$* znTwQ<<8m)~ox+|Om|5=9R?8jwn%dgmXS+9%s%-0PYHqjpu$T<=+%b-cIWcTLE;PAx zCGFk8fQX2Q$(+sSnYx<`9M&`6vlW_7QXQ}mZJ>6=Uoff+jZrB_c%nIqag)5tmR7YW zXAN02gh{)N@NLlX7$Tn%=4@|v_15!+p;E`C998w%&MYPMW$lyYV?Ymz0nsy=5kt?V z?D+OU(2Mx3I9}7Y|In>zo*Wl?SQ4t$-IaktOYN1JPJ8dk_@N|Wc8rsltl$Ta!k(dH zl?$AnA2_YOzv5~1#+e25{OLV;_|1wr==T>i(Q_82T4RTj7w#GLyicqOqjEQsxMogCd`J^>5e2)W-O@y31zB}pW@wX2{oHBCXQP|nphenjHF0Y#q5fSOs zl~j3=2sxRYvMU#Q?;a>_1m7V~MsZx7{at2PY`ZYTW!*|>!jjZQ zX!fVx@R$?{L#oB99OX*Kt|X2#F^j6=fP@^3Ii+V1vFiu{O{uCr zj{$BJdb(>QA*7^kTU_Ks`ykgWZd`Y~{|L+sRiWwD70T69kpw<+h+xRSqCoS8G}}{C zNVD}xCQDN;-2FlMIKb=~N%Gy}%N$o$0Xu`<@Wwzj)uGy()Se~{E|YKA-L$ci=Fzkdz|>(R<*TxovFeyR)-2_*x{bUv|7Fx9e?3@CG!s6VFLvXtEjb>}W8a-G-` zP#OsKu(;5+Jey5yKsultUP7X}hTV#BsUeOG5#ZWZs)-p{`I2y(K4h zy3FLD(6SqIY;4T#g`sowSh4oAXCFq2fJx0TM4djZ_m!J_?8{*?F)^)kKEGqPLhYNZ?!jUtdejg|%W)IHa8=J-h6RR7=& z7lF6R8ILStS-8h1qtmiajtgjt^_b3;ALKAeoL~UBX9*aSKm1;B1g0E&o`%1hcS6^6 zxjkkVmD#&fpb-`UNXwaBT{gViz80?@)&BI@I>%hn2>GZt&IZ*+9w$4n8psb~9mQ}z zcZJ&lcsIkZu}BtkVx;3A59xmu3Apv6MNZoI_;=pOjR36aJj`XF&(W&(=1Oc`N$$@z z)ZHDFjOK&sdHi5I{%OQ-WdTlW$sQL`u{zgZ`X!g={o=2WyyzSW^aA3(t6Xss}Eh{V%~|Z@vul=iqlkmlmI*oH@k!8Q4qugIWNvYfsVc zm$sxZex7+_pr$8g#A{}A><7V5!)F+)#&j`aBGXoh`x2f|f0Db3E|3;sN4u5E0{Cr2{R4i;j({r6 znI{nJ!%utU{{g>&n_18f4KG6Gw4F~ist6F&iG>Xmv~vYGZkJC>>R zkJIgE1Z5U@N(iqj4^CNlr*X&SL!^+4q24dmOJV!~!S1e@|M}-S$3z)Y#cZIss}FDR zA*IYC@{W$@xrW{MaK2c5w+1RtCV8d z;HC9dMu@auq-$a`pWNWN4`hOa={14l6TUEQ|;~rwI7ml`f{^=OOrJ&LL2TG7X_l zR#yE=XS9;~jlbv|608FfyTP)LI@lL6R=mEEIfR)F z3l@~gc1^l;-0(pp*%#;mp_s}tjF-3?9QPRzuJ8WRUA$TGnd@!#c*ALhdjkr1C@u!T zm0+<#J`Rag16i66B5mYSWNSIa?A~6aRDtsyZx;L}xMFu_tJGC%q307tv%a@C|25o7hhsj8`xsf1Dx&~8%gmgM*t|a6cEcuRPvF$+Kox4oF@t>cJrBg-79oh zo35Y>M@Gt`&**xlv>A?YHkvWM(FaUzl0LkKDbD8nIb(agil=D`}g&X=;(^{ z33GGv36FyHb-S>WrX06~PhUToJu5pctsxzWv?ZB87yH-Pq1ignhOpP*(8*xdrhiwD zaK*4lVL$-EXuOJ`;A<}>fhWi04?~?{?d(kB-R8I|}w&`D=zL_?pd!|LiDTVj5XiMgKt`o@hLAL?IL zS5@`pf9$w&s8h&)qSe*cH!hS>Qc{|bJ$m%0Em7iXZ9_xA)YR0QHIxH;d^9mob3fSP z_+hp8adbgJkCVN;l++KoH4%ow?{@aVK|wmHHd2bmh9pbQ$k@DImRep+cIVEW3G&A8 z-yc>SJnd){_kjVRh>O)vD1|814G`8SsmYelmr_1@6eG&+S)CcT1`?A37%YT~|FkSW zj0W>Ywc!xUbLS%$p_~!njxZ;^bV=JsP)g+RDX=#elTuQcbYyk4wK0{4AIf&OD;&PL zKO7V&J^tYxK3)R;Kc~byucu>^wON2!G93{}rS|z_|Z{0FWwNp_E-}uVN!!x9Y6`OZ7 zE3C)SIXN9y%oOG1Cc5sC>OoK|`5S7`^*r`xakG5g%7b00si~{#%K^n8$$fW>G2SIW zju16plH?jP2&g4jLQs=Cf?7_X?9rVC(LCn^f`h|`ABjAEJdhF^5OBkJbJ@3{ zLG-P+cSf_=N0Xzu@4u210s>%-+H8!M^#^T_6d<(CQUf0FutH?9l`D#qZ~9eDOCAC zl@31(!@xg=x3-50lkF;pZ|*-h6&yz|`qB${3XnVlD87T10mh&CRu-lLm z^dIBp^XJYD%;ORg>Xs>Z`uO0;VBI?6DiuvA@$vB!d^MjxGvy~tAqiupSo7UL|Ld{j zxX{H+w+vKXxOt&&mp&7H^yrQnlg_heJxNcFdB)@eC>&u*=)5zt2Hh{d`26TfbUK>H zpI6tRasvxcZXj5IRL>^3eui)9K*hn3(0bIU_-=_GGZg#kNCaS>UNa6Cb@rO1qM|;- zw1TYcnB4Tsm#CH@zj}HU9PhqUIwDyL*THEW6Bl_pYHD*@W@_q3o6FM_At50bNlA@s zxv5W8)k8l4Dt?Lh$WhSVd3ZDi`(C!)R{m%Ma$ukHhf*pu(clfXN%9JI6|Jv0eltZ1=LIfHx^ zQ0+$^@Ik!XR#0BOdNslHV-ux)MOZ?D6;zUR$ihv<-yVrZF4g;}C{w3}`o>1w_BicokKwDHUzC*E(V4sCm7Myt3Kq1xB)16EF#i~ z{7ze2+k~YX_LhTB3Z2H{Xe|$4o~k|3lX@Y zMBs}5t?!mfi^XYR&Ab2zb)G_-apU8@q)}Mo(02zq@AyZCF(loL)lpvlJQ)>&c0P$2 z2nC>tnLGIxO6BqE=+ZS$-X2ivPPtQGR zJkMtEFP+dB(2oNZh^x>&IRtcF2L;Tqlp1L=q>Bw_^2mPX~DWjDOp6Tj}$jJp|y%JGfL0LNeo8Ok=m02{{3{r&yFnc2QOT+pLY z8UUW5YWd(XA0Q|If}Vu&XlQ6Aml3?IZGC4*CIYR1Y*Jvo3-^e*O4YiY6MhGhTa6agYJ@s;$snZ>ciN;ZL}WmSMH}W8vt9xF#wSdW}o!EB!B76DLrNU0CYuCQ0P;uS6m*;I*KX<78Ht7*sE1w2vtyCjvX}{&^_Ud()_$|3( zI_OYtYydsZlQET&M>V37R8cW1Q%O%sGD~$%TVzv0S_|Bv8-=X&S3EZk&tp`l9EEaR&OKh8II4q+1gEr3ZhNpqCpJi7)6Shq=K!KvH~L<4XI zfrQ`wDGdUrc7!15aYwD@G66wB1Oa3c-+TG^xFaq=f|P+?3&I?_l|kk(Z-$QtE7e-# z>Oah3OA9baBuYMQWT-|rMi$169zTBf+u|Mt#W%t9n3ThDjv&<5?`g;!@;kveEvODP zyrkDDr!~zsY=VC`SGl$q5qwR>d#C5qXnes-kD`vfSO~ zbt3`;wHt2J(9l?y&`aW|&h6McS{ z3azD294VEc5hk$!GKskoA|n2Py-oLKCpIbxTJJxcybWz}anW;u zeX|&jhmWs?-6R5FW<}F?BISTs6FjKColzdQZVes5#Sq;31nFZBmK_%pB``41FCxPD z!dI<&f8i<#*Di#cK$<%!_S=612;9xJil9Q8jl^9J9gW-1gHOYYj4q?2qjR0N z4GwxpU4z~-Mc6UV1A1&JcD{|;`)(`V|SyWt%Y@~Xl*F+zF9_*n8avmcl1wNUD*Imp z0wOm6jEOxB$nmYBR903_)c<_b+1~b5<9OtKd6MKsMF8#U0nkDL z(9*76N`Sgx9LUpw>~KR2ERpm>E@RT#RSY;Y~?h z$d*_BegqQAE*RVniTPPj{US{MKAwGr^7HOHZ|^**`a|a35znWs9S=+<$&1mRcX14k z&mE7aI1!`bdjKM7CVJeD9l<gaD~JLsG3pAD0^!7a2A2-ftuV85kiA?^K-kODS2B^G z|H2J9lnkawuhwS-UT}p)B;LTrT`*Medl1c+yYWw&k4gtwH4ML};HSw@ilVFm54oe2 z@;?a~j2Piy{y|$HfZQ{>z4X-2Q)Qk1+|Mo`^va{c)ZnT&=09A_3gTjO?QLy0n3-i9 zR%i9IgQQdsIHMdQlA#v8AIZ?j1EgQWW8yED23kX2AX;44fw5^t*?g*`tHP*O@`11e zZ|6n};ye&cm;mD$QW$_5%;=v(`vI!Zv3+RcuY7BcbGi@-bD%t^rV(G*jqS2Cgva-V|9Yvf*xPZ})29 zU}4FP%zrtvZ&Tb@VcXkIqYIl5N0YHKHK-=f9UFUw``3N~Sv3H~c}17s7oq`q5Fx*L zv)ayARZ?<9?sq`}+me8dlT%44vEY7D!y63*;53u~;KCjP>j=ne8l$U=b5LnVRynk6zBou4ljH|k@4-f-6x)!~q2;xLU%-QeWy&Is3c>i95 z0iTS7#PBGjM4SbA0KiRp^ysa!h`*no=XkkwF2P`a=s)NuU-UeX0(es-C%Dsa-;@sj zZ&RwrhGK7TUp3r)px~H*f-_?CN}&8(JGprQ~b$JXRhDT}_7drFo+dY{BTF1~4v=EMN`l+~1t954xIXD#I zDJ3Q(DA~!s13Qe6u1kUOFkYR%<3PitE>A{HTSbGFo#=`N=gRU|IyOYTBd3qpl;;9f1+*&MEx>`cOJ~` z>p8CO5VaF}F4_fiiu_K*lXQfdd)t{=7IW|KjR0W-)GpX9b8LuezC^0`KMSi&{_`;? zigx9nQ8b`{fK?5%{;w1GSB@A{E?q?r3!0E<{}+QxI&;L|@`-OyQWDj>!fB|AOj-Ul zHLz(QyVj`=Ez7=l@GSGtJ9w^o=pDeg5!9_*=pcZ$9r)n`26O}I(Z>FHWC_nXw9~VoeRq;*0W(_;Go$>+Jo!q2@yoDPz5fdE z25HpgS0xPiYCy?#jX_GT(h2+*_r66OVaG5CZqd`Tpj!a8S%?`I7xxBY5~}}H?^qH* zb^chPrr%d_ermb_CskYps@1Qcrj)K;px9sJqcD%r?VF)vgsfs7K+R@>AIL2V)_pf^ zNs@=GZ5W8L0Gmoi@o$@Y9aOt=pm2mx0o%fjYQ<})dR;{~6lkHsCmaW!Xcw7R%>KTy zVIC2<-hHIaT>l?PzXsccjvVO_mxTQXJ8FX#c%=(*JV+=Wjxk z1y$abZhG`xcjJ4BiTXLBO;+Nd7QiZ{DX&YtwY3#5;ejQ>w9RCn##`sB|BbinGGGar zwjzMl0cw0%puB9kTm6K;aL%h7}773r!l^A>Eb^bQ{UjrBf}i{>T*{ zF4$6j4Ra`xE7^^*`I8qo$>%^=E#)F`PtPL#{{7op^~pTKmBNJ~yi=r9;GMF7S+zgaTuHBdo=zqUSH-iO=6)YA~VkC&8| zc2|D!S1#KB1zNZmQ0oXu>L$qe0c8zRov+XT8(K+Xf4G`I#MSy|hC00YC2MPIa~0|5 zPnTjstfz$!ictvvxy`yF%w=_KHEAivw-lGq85 z^$~>*PW+KnA{%@Tau39>|67&eW`SyxKBmX#ARd8WYw(LdI4m>%hl7yhdyK0of{9fP zV-=wm%q#yREewp7+Y^8Txku8FCI90fFo!1{m`yzKsf9nP9ZMeJ7i{sl+HyaA`m}*x z^zGZXScI^kzrr+N7dk=!#foUP)Pl5^3kI%ihmXsju8qd$Z<{6ORTf?azxI~^-306) z@;pLywi0!DYkzg7M@t1+^w?ND1WC&J1gWpWudg%HuUT!FMri(fMtp3*YJW=>>4j$3 zh{!eJJ~(g%;x%PtWQ-4LIHD11t9b)pixpT&CD$r=CU6A)&fJ2P$4$Bf)Pj=c1Qqn$ zQq;ROEkxair0k3r$f7&4cmhQXGN3dFXbD5mk`dmAmjAz@l_Un9+Sf6m20ay|nO>1_ z)N=s8jERZKXumS#VI-8Zhum!hC%Yyvw8ZD}4{&Wlxw>J8mz&Viu86vgYrz%WB}u`8 zlJg{gWfi}mStjdH9NAJjK*S``g(BG^|k_E`VPB&?ToNg+!%AEt$98O^XIWTq?TPljIEq&|7 zlXtDZBeNX<4J3;Y?hWD(-wlXkW#xxxsy^UjfJR4mc|Gr!uHx?|nvJCnPI`urKb;l7 zUTa|~_qmhGv9fL!AijU>xd&LdzQ`TH^rOZ669#B3c~?>N+D*k9{K!jwK3xUgf^E;+ z0iwPL)XJSNu{T?}V7PDPan}LQ2~9i-g_HXM9c0+kp8;DLA2i9(e2aJvxyGvsYFolN zsfN)rjM*^?e|ohgedJr%V4Rcs+V`WnE9)U*3P5OnV9v7a1alKxYjkH|W%}kERdj?{5cyB z6+c#7hrU2g4K-QKf~vBfO8#ltsP`T+FrSt6pQyc?C!voC{ulk$bf z=7=A{k_NPs--g!f!0$rEg)&FRzhl9lJGUw!AYBHqQAT}q2YkV;5{&cm3+XG+2>D)s zr89Xy_&E0Hmw_@_x&Q*F)q&CgC$LbkCf%~{;iZ@*QETB1L|we((VCWILCkF#=cZJ| z?*jsToj$9{c^a$sKQ4yo`Nj~=M{SJ7qVyI4{*Hu_fl zc=XarR)(qt*||tqevbXbz+IneXB_LuA@K?#3pNYjD%J?K;^G?Wq6b@My(VRkI)EC< zo){5ckJ`bd7(X_iw}71L0aUmPHW z?E$#h69bf%L214e&PbT251_}0^ahGjUW`#P@Poe zL&L=daaM3^TG9ge;!3Hzff_i18%iRLf?y@!vpZscZ5ltOk&67aZ1+|FK#?UcxWYxQ zTRh_4xG-UGMW8ipJrAp=d*pt#Q$9qGw|x32aHmm%uXd><+{h2S?6t;pOX1=`Bm^ez zEiWYsH@-m#4@MII_$gck^vcRwJQ_ZY9d&T*memsn8D{-$N=Ql56ae3G?P{(N$OQn- zQH2k<(d|uNIk2u!Y)9G1c&);rKkxB|L1pk_t7H8$fgk5K^467mOZ2lpTB&r8jL)p<3c&48ezR)9os#a@N`uwfEt%dT% z(Uhx{dn-IE;6Hb{TXSEjfeD!S@r#BUn7b=U!yz_tA^m!>e0$LyililANO<=!y>wg`ziQguaO_ONF$uJd$uYfLA1^ie^9{{EF z1{ch>wyJGW8Y8jTtkgg>v}o+-P+0V zS|N+EHFZ}U;c?GwiQws_1d-IPbQF##|5?ytGHbQFO?^C)daWkRWBA;ExrP?CWGHR6dIWH_a=4hyu@Q?b83^#AE0lRnb?xq}J}*JBoud+m~y@@d^|%@})y6 zNrGqngB;UxkKDyli4`We!%N3xYtW);%l9H4U$iFliGgyP3u*FuSxO1s5mfa<%9jJq zX0yNe`MI%ng*$C$dMj$oTe+$(n6~UfU5?gMp^%+lM(oSK9&D@04`YRs0O`!v@0Pk8B z_Ibky!n(<9nbyC802iPA(I{svTk+SqVB7V4WCr%`P)Jr>y zRHVanxJ>n4T0i;lCD)q*^*n);nWYp`Vxa#H{M#8hcWYqb9&9^U8Fp7}_I=bEoB2ZV zP3}PuDh_bfSjmB8qQK3#o5J#q0>9Wc1kCEH`@X>Gkyzf72T$Bu$j_v!3#EUgdhweD z_+VC6m;FbO&uvFT?E?Q>iZdrfoq;q*?v7;_Ozc?`^_{kiTAt6A!8JpUZ>PB6uC{N- zT*@Dr>^)1*WgSo@25tHks0l#sI}3cNGQu#iqR2_C(6ik5IYEh{2F)^QoBZZ*n?z{7 z-n@*?q<2Ks?iX|S%7stmzl(C=&&DdG7F&z25_LC!U{!Tc8Tw|ZnCiN%_`Xc(YNB8j-GO@hysR!M-P-ciHp9hTzxc&Q`QiVxqpbQK zS#0D66O}uH3zrH-%e{VCJTK=tpQ|F#olc`jnZHU(6VDq+d^Ve`NEw@R@pgHB3o@bM zt`Y^kQl6P(p5Zq>bjP&qWy=kt-o(Wf!pf?p`5sf{sCaKo((#yj20wg>c^WAxDP&i* z?Rx{?Fn4+Uy+uIv>{zx>FiCSKVU#n&rlK|1ma4zR`|}QJx2hw}L@TzQyGzTbW`ge8 zN*$5oy?3&PK9xR}3wEMZzvOajIQ^ENx{C^B`Ob1_EAku;mjd>5hoU$aoNL-;m$L1% zn5ZOaENru_l2P~vXJ%~*V^DopV^|!x^r~%V`!4%Rj7Yq-5vM)x0(%H^TDjD`eW}U& zGOff%vi?+R3)%2tSFg=WmrAHXTT9upJ&RXHO7c4>BNgAg19nDn{2$ z7wF3~WOSS>wUu0%7<)~r5nQpk;7aqX7F}Smu3toZynFcVYZ`~|Ir@~rIne*?QhICKE7C1(6my57`r>tb}_qmOdb4IX~#_@6pWOIH}t$RhGXK zM%vsb!&pw8Nxxy{qbn9Q+#%y1Cl%|AlALe{>3h5GkI(02Zj3ITztFD@_9!u7P+^Dk zXM;bb?a&yv%e>K`9QA{RP-7Q{Gew{3ADzi47M%~dtrB{#*Ro5+_`jFZ!=WnQF2F;l z%;{0OFjX9%Sr)Fp*YW!gpE5mWHAt|y{PFBX-*ir!+RYT#@nA8|xL=u%?U#bn7b|(6 z@DC5XtJo}XjvWi72(5oR7c=fQu_e4zwb%LNL4RZaUf>gWe!c&&aS)+v-|Q#vj4d8? zPKw?Qhm~68Qh~FslJlKfdpcV+S))qz`sfcuwy3s*JLboWZD%hkDi_KG?I=?FxMWMo zrC~M!RbWiHQ*&8Ra!1=fFvqmW$4XCbXKk?i(i4kHW!L(IIi+tEKFrv9*I6m^0!FoB z;{-Z}1d-HoErYjnG~K1D6J2|*v|TWq+u{+*e^1eOHwbj1V5;k!oC(YA*Ip%rkM9iU zKjK$HXHT(SDQDRo_$*_xk#(PEvwexqT)nximNaJkwKu(hYqzdqA!+j`o^LYpX~Em- z`I`@07s1t@DjazxMjU!X$~LopZC;#Xl~-BopZj^izP%F$px888neVwTjcq_(#mDhumCENU@o7pLtm zt>O|{Ah<1Fx68Gs=cAv!OhUIAGj2-Jxipb78sB7FQGBec+@!lfjB5l6tonVjpRYCq zrP(VAR?JTSBHYz~GoN{ZUjEyH^1^>K$%Jus&ip^AvxfF6c6Nd?&)}ZDP^#|(L-Wq- z5L6(kvBaNZuOHe|*i;>CX4~24hxSJD(c{zhKkl}?UyH0=tGRjA>E{ZOMFyNwUX3o_ z>nin~PsZH$zF4~n-JY`whcAS5g1ep~uV|}w!KH~fY(8{vzLqXGQ+uF6Dz3T}V4MhO zYf$6j*B6$a73&Mbs{`3B{_7tjIO@gxCu&+%{kCo1yso=izVdFYA;jqQa0X$nUb|cd znN`?b{VtV>L35Aa)u4&*%B8&;eCFzUze&sOt}p9bnKHVCbGy}^9B5! zLdN+irW+M>Lt=op^@B0#)bHBG%&`e` z+xo<)V7rosXI33=Vh6j@bk$lvm`~d+oW2%kS*GxyeA+U5yzG@k7=Mc4k2B1f0m_@J zeqq%^s<9PH=rVdE&hUurM*a&KYm1k6*_+y%U+`#st7`teVc+6ry_f`RW89H0N7gIg z(HhQV@Yft&@Rc9lAdhBP?UrZRa5^iEjAg3J8$iK4$veNI*kZBF9(GSBYnf|LG;_~p znV@S13B`5Xr?Ak+vvfr;|5tb6?q=_%-G;mc_|+1Z%T~X>Rt(7#UvpHNxSbjGst-FY zoe3k3_nSF3%o<86z$(!0l81M43l`;gFuh=Km3J=^vJgXocf1I#TgUGgR;d>hM4Ddc z*8>k*$rEQj6i|ZRCMn`=ix_2)GxgCDbr(}aNZ9n ztYmU6HVTF;31|%0E8iUwxvZWQb)_@mlb-UVbE%Z~TYb8jxatYg;%769){E=5VcSw6 zt~{f0c@~vF8rOPo`Zu0p@eR}UUpJ0j3zFfG#aZp*Q|VtYHgoH;oY((kF#!{6+wIMU39%@7MP@lOjw6pAqM6`R2@|<_u6s_T_p}QRrBi@)^l~c=a@% z$QR4J&77;OMyiq+(&JUzE6$FyNBS#9W#X)dj64Lv3()7Rk60+10LV12)7py*bDW}9 zkIcB+GKwBni4tiw@TJ#E5yDnn(5>Fzc~JEq^T$%sFK4^(b-aRwuG+~)Hn~*YbkF=H zq1)0F;^b5Ee(6469QqY2H&JJe6f4e+hdz_Vpqb8O^ZT`7`{4rj%m=#l5>cFD(oGWP zob=(E#(ZAbi*c-rLjiba1pkwdaMnx_EzPm2Xff|>Ix zsRF$Irs|^W&zeU(yb?{%OPMl#f7oINCusJa`79U#w*Wo90z+5n>&0QYaEqU?q$mqb z2_m))HT~8iukQAh?AYDclx%P3hlQoL>HnX~fC`LQmgy6aLzRhk9&+|pG&H7_UAoqmnejBmsx&c)>P5zKUw$ne;UDiZcP<%j+xOY));tfft#O&# z&ZPRG;wq%KyA`!3oC(uv*c#Vv_7!R3=8krR&khXb4~*O1c59;PblWSeT=;K%^Bg}* z8X5bBOGwmC$uNhW#$LUtYb_yVs;A`+w^pQ%0(=C?wfS>qh0?dmkR`I<-`Q7JmDRcV zxAr~~X~oGmFO{54{juq}lbdk=tY`Ss1|;m|qLQA_pT{W;2(q;Pkf1L06QIevT4veMN8#=b>eL+U;PVm+qd4#P(q_)nbw%dDAvH z$4Euh;faX-NeN% zX!Y7U8XaaGEgkL->g++FC#;V$tBc$1rf0ZbPT8Q#+_SFd#pB=oxzOXk;C9+*3~nV1 zRbA+1ly}mf*G;RZf&2ArE{=}tUapP!HXopY0?(*bjc>mrB+~R%MQdX!HKSa*E#tBO z18m-h#H=QCu6rB#13b?>?-+I|Yos%$>Pm#uOK5Xz|C+uqL*hPgOE~x567myBpto@o(s;uLYYm{Yac=c9~I3tjjK5vnZolBG$5BTqW&$q^%7u z!k*!@*jwIeY`OEHw`Rp2$57R;bhvZJtGBL?mE&8{Gv#L4rx(dD4(q&jfA9g&CocaR|+~|K=xD~{+ z2*>FZnV*@RNImQI-oL3-A@r^x6)_i~>!xps-_H9qmC$k_b|C1@yiW4-GtSA1K1-RU zUq1K_&bz&clNz0w!?vg`8t=IS3Ib*hzGr^d@r|3 z;q^wTcD>iCfu3FeUIuf}uE{6IR3gLGHvSLvNQSMHKi|jqIRLpFo z-MicM3x4JsuPpc<+p_41+}3uOiXH1u4slijIEt+8S8t4uixz*Mk8(QMko4agToS@-Nn{ z{#J&${xS+GB9UA0kJ;|kLw)j0ne&=(X%QJmrkc{ zEH|TtMVH?jFN^bnAWl=jay{dF?7632#!r`tww9Nk#)jt)XYM-I-YHhqMRVl%xJsLt!bM|;nz@@fcY^e~9_Vh&SPW+dnXm+lcJa|AJLg;( z%=>hu+%&+-pkLt~ZJx-cAIq0P(T4_1-}=mr{eeD{01nCLavJU4sQzcu3;mv1lDgY! z&JQgMZ*1((rydU*5mvAGnBQRe8IG?n_FTNOnsZ4s^)ggh2$9_ zit4V{%m7zhlJE%>!A>5!E(_nRuOeOC_6dt0Su>un*y%Hk0A3TrmSF9jPf}HuADQPv zG)b3z7#R4Xs~}f}Y9Jel6Sb>9&%|F}E3x)`AtP}hZ+>+llX=ra{oWLv?9|*eKodzf zhgOGdUfP2T)xDXlea6h&tBAKmIh$>)O=9z-#n;fTaP}{! z)T~HJyxVleHD5|>a;Nr8a{d!oZt7gRmI)py?oWLQW2}tHA#QoHvIV@@cf?gb5%;=` zU4QY-sE190Bt?SFPHAuiilOSolI7xy>~y~s^snSjmU%XP9d`Ysk}wd!V7Ry2zTW(8 zf{RW!PIdefzDj>4bGUnxi^5M%s)C5jNd|{yWPAND&J&_W>abh}_1&?wnJ;F+E9C-A zf}Z})tI00N6)-nO7@$ms6^siKJ%zucs?ii{1)TPiAYHd1c*+1pp?cO>UVqV=S z=r-@57gCgOx!0fYpR1M5T1FL50dbx^XU?{-|9tS)Rp+X?ZQG#Bb%os*m>960xf++2 z2gTOw!G9QbWt}Z=E?xBY47t~QVXp+OtE%Gpm(hkVXUcSZ_7=C)nG6qc0_R4)`{d`xKi+=biG zj_2L{;dK^A%hx-W&bv9S7VRs+=%wm~hR%kI8_O%q{JJo8)@R|rEd_Na8N*q`Ij%Yv zREv~tPdFA%B-|0K$*AscZt%Ahm$2dNX3w(O{8J-Gj^rbXe8FyI zt4Wt9T)r~PWdy$(M$Kn5y`9H`nP!~~x5P2H-n?7TJE_*(B;ss1=+r1{jUBb3qMjX3 zI%kFbu5=FH`F`;&Eh3{q83mj{W;>%FO7?KVHD`ZST_ZSIZM$5We|6akPrXuocg14# zgqJ6GU_`SmHk0jku?QJMuik9$>|DzhudRtA`@KAe+j*Ia-->cpUdG+45;duQ*FS_} zD1Rm*WM@$#Y`n{7AXu+Eq|r8_r>ci{5KkGH6wbc2r7BXU>1=D0uIBLeCWS{8%FfpJ zwylM7Z}-JptCl!;45%smFrToB5~mmVW>?MU)7&NdxS5r!`7=(pu2F3zsV5BEVJfnZ zm0)|+R)0~0Cp8=#!{`DDOl3H=4MrMmU3Xp0o>zcRu!~WS%g7rTliB*7!8VQ^eJTTr zq52o4Bv@|d`(AZKz9qzIi$HOC_WC#ZV8U{)!L9QbBfPjrn}}kvhn!d34K?OWF@M_* zJIJ41BYPS^B#L41M8WX4P(9szwgiv=?{C!}3_f#CQ*^6j@+x$bt8CRDurjl`?sFUT zSG4br^+obwrvmZ3GcrM9a_6k#jR?OR?UUlcJ5D9>$}$48&8{l+aSece~tRahT_hGNsOm!4-{hsr#Fz@u%u1r=yN}AOKAlK zAn%ljhW@a689mZP{4uy|$no1MGrb+5m@iYxynduv%LzTXrLg*Sd*M<(NFdAl{(kAS zDjb=o27kMe{0>RWScEp_3diHRkd`%SfczIP~L;yIjxw-F| zk`{vaH@?|@zPD5!3lJlm6KvB`&bB!k`TU{Qq?x^1j9Kuaa56SMOeNJ*x(`X7v^r>g zGd6VPo}heVn+au!6mIhttTSV&8cn3xVVHA!vTD$Vc(+}~&{EF(VaP>dWu?ScU`W-P zm32>B;$lhv!LEUV#F&1=fmOn&FnD|E%-OSeCsr!)m!4%|%+2@CjQ}jbPLG!PbLMAy z9n4U<_eULg69r_R6G5BqTHOklu1rHYZ_P6F>B14Ov|?vFmJSw;A(&V5Yru_NIKY!| zwLNu!iWr`EWP0DfHzNYQh(-Y^3sO*#(-ZQC*DjM3Q~yqKDBhSJIUjWTZ2kzz^EG<< zkMs_u4k&wW<2a(FZ5T)3A;EXfKVtWD(+Rhr#oXkO3L8UDqmvL>Y~!d>O6yq?wb0#BFM&M=fpOKHVPmo9jm<^WVCLH1)V}Z3GN5F4nr9g|~#x+XlaDp`nQU|6YS`7|f z9@S;Mku`J($NHXfMqxU9cPhjN=j8Tfpq z!=SPA&=@Lc_T(dszFU+)6AU9o#`9;dEe$L<a>lKClo;f@1u<~SX9 zc2Y=UJ6NSfcVJ$%w{orQQR+xt{(6H-)B<3c>+;FfJmyqn{y-)Xdr$SpnFOw+>i{ZJc(cSi|?MNlFMG`BgQtz}H9#q=gy1yML zHmV((TB9!6*zlDup#~&_DUsxDcrojMfod9WqD$F01;1C#2_ zT?qeu2J&ZNc)%Nx0XALF8G(luZ}aFNB1TUwM})k~t}(6oJ}qEOmYBbQNKjarwC+RN z&F^UH71!dS=WONrgO|?o3rUk(HSbY)6aj?tC$&AwbwX6tRg-T%yQUSRYohzPPwLkr zSLEE^ajpNfm5Va)=ocmp@UYmAijx${{BBVem-gkXl{^y6CRy&)j0F)n1)xL1pl>&a}>F+DxwL z9~WDax7%?Z(d?|*VS(wJ;OhALAHFr+8+0c$ZZ2)Sl#AAN#lahqoM`JOx14k>pTl2Uc}QeZL>X77W~7 zdc5E*p67%! zYu+=XhR46OsUllj{AljbL79O!JE7MD#HGcM7t8I}XPvbusf$j3INO%&e-0%Y9df@G zgHA!N7y3jEATJHuZq5K(UA%l9Ew+ z8x6iaO5rrDdM{~sR&;YVba32r%+n&spE^INg}Z)kyE#dcT4BCB~S)pJ|)k7>6*p~KA^eW3P1RStT!Gia05x4rPfo->I*r)TwY@2J=?KJ{^^W2e$P(BhysTMOiE(oYff|m9Yp2&*QJ|?&tKh%M z2Jj9&Fu}*dv61BKr;>J~Rc`hvj4f?D>(x<*y^HVK0(ErAKHJST=t{B~u$Pnje%f`r zmOJ7o(t%%wrnghf>N&u-id(smdo4nB<>u7_>TUM@na2z9L>LT5p3&u^Vq-YxPs^Ia z4$FqZ#8#%g1VPz8hgWzdh8QB?As>)mke^`5quW!M0Ko@8%-OY@Nb7O`lZNK4sj=nu z4bgCDla${S%M4GpA5F(uVEgguSfRvWY9p&FJ-j(fQy+jFlzD+XhrA!M;rFhloF!+y z-JZ{QM*CP$p`bo;mqaK0_>B_*I?e^jxOrKh2j~T5zo^B$DC7XAT;Rrgy*Y~Z{FLDR z+|OTEHw^{OJ*FdDKrS(Iwaz-PReaVEvBgs{>*Ct}r$#n|!rW7T$LH{6>8Ib7rA*$C zR$p$KvcTh-&AQ1+=6~>~`;9X*5j%Eic5rq5As0izrAF>ihRXW+$uhTdavjKI&X*+* zn@HC=nuY5Ou06K)GIYdJ_2K{p_Ju!qwfNo_bJp7suy$S1Am&@q+S98Nkxm4wwq3l? zP|MgM(f0f%aMuhDszsV#s9bxVq2lg(tlb=_s*raq2+usPsN-O&etzmA{$$kDk}ZZD zb)(A+^mhr6vA;B|kuvt(VZOYDu8ef@*UWCsH$&!^4+u)|OTgD#LTMi7;sZYRl`Gex zsy_f@>gIJ2@M6a`_XDIyeV^vH0MG)ZD6o67TNjS3;Uv#i>h_ww)|_+jWXtZzay zJh?V%jzy}2aI*m(G~|}L>){*I7pei0W!w#QF$TU}_ut)y+X$|6^4$DU8L_>K=p(Nq zj}~%iC@KU=pCX7&U(=N;(Za9u!SgAES_5qs?pF`2wbsrtA|qvkemhY$j6x)OnM@ z(;w_Y`^0Y>`P<7)jY4b>0A@>y#^OtP$H{Xp%i%+j%@&j$jab{3#;F_<+z195QZdDo zWUlW6`)Sl}txOFd6F=y%*cOn){Tb+g+miUJ$^T5r4mt>=Y(};GJ#Xx)j)T#I*>e^L zoJT)El|I(*163;e)(g4$#Gf%K-(Y)ZfglfoIMKM>`c_o;Ql`!KVn<2dv$VP;lW3XL zxBnD-6R1Ba*;5F-o?%1-@Sjlw%C9@N>+q-Tj1enF9Qo~cmEji38CrY}3s#L+@h(Ax zr67{ee`p(m`-T(iSMIxi#Eeq5R%)kz^Pj}cb(?JUL0ZaJC9`y~fgdPoRSLtdKobt)E)YT#Z|Dj$eYY>>#EChM%DP1NiA+SH{rMwLFDV zXmc-g(S1K2vxjPHe+&x!eq{iJ+YG2n7Xr2sBme{S?pdFTn?AJmwu&tXbQ`?9ABo3F z3jL)QnO0+FUj_^3ysPju(>l<=RR~qIF(S8pfD$_@U1C7&DxZ>7xb&%dZ3c!O|BBr4 zVJUFXK`LH1 zaz;c=fb-8U`@mY4JlA_eLua2^S+%eGZo9(1BXh`55*zvgB?F>oA|JBA=3g{g_P>bR zN;kbxkZW89-a^&Gb22*ixxYP2lGCkbE>g<1dfaeWBsY0q$KKnD$}m3Mf<<%b*AH-Q zSWrNpNv)JMsBf##*zNKdOm%tvIW`$`CRn3^0@jW5_EYFLCtYyD>7M2GfJh-Lwi5N^ zr53oCI%#}tI9F&Tfx?B3+%Rat$x~n6=w5t!tgt*x>>$p5%j7{Z1_;XBQL0C~l0cvo z`yUz^q$kx}cjDa?lyTycgV=F|da}L(5b@=;(#U+*|Z~E1G zQ2kK98po%ebdGxcP)Z;R*m2ozp9I6af3_v%>0NRv{M~!N(Rc`iXe>Nfd9yDeAip3~ zdUj>GsM=^Zujd7art=K!Qvbxheo!)Bry1t=p5frG&Mrm?w?uAVgt-^To2G9aQjf|2 zc94X-w#`6ZDlofJ9G^1kKO2fe@SR0>Ez<5We8`Fb`aev7>iU2pbg>EUK)^==zmVl~ z5KK{)+}NE0fZQ)t4%g{iLlnv;r9AaU;7hkOUgum9jM0p+&Ee?!T4=5=CL&GMP#38E@rdN=EC9_MWum_8M3afiQ$)%$IzIZ z*VlhkKY+w2I~N`L$r6a9@^YZORzs}WiFko`{~M7m%<{b&;M)r2npjk-U6Rj-U&Nb({& zE&kRP$)OVM2f4p1d|lRlsj7P{6E~J$OY`{3ApqBfHV(I%PVi2%kdY_}PM5{-g@y=k za1gl}l~LE9{c>t+fZT|vh#LMF)))X3)G-M~rQ6)FE|=dlVoq)SW+XiFA_5Q5-`_zg z-_r4G!Sw5v*x>($Z+u7liO%LLI8LXo{I0vkoM!?ij%S3Vb|=oYhQAWkRF_;uHo^|- zrS6ku>c?VR>VMW(G~_D$#{O&-cu=}58mh1lZ<~PpmD@^s^s##Ei8`ecJ`97;cjlJ!+ z4Lh}6=@Dvpri;2}W3$hq1Qy{#&9M3=o~G>9Z*yhd4;s1cZcUEl(;6qw@eRN}I>lS0 zMSGjE$ur5KX18Uzwa1Z=tse**B1D{_-q1aXzT!q}aj+pcm?X%F=+vNC(yZ)U>|nKk zPo%N5lBtnabjW)YjxhH>>Xk0;^#U5^0)ev#^Yf_O9CJn4&!6$v13t{?VE!U zTXjc_%q)VZ`5lHB=Eh2R;Hx7`K8yZTi8Bg&oIqrH`)vQ7Y;S7y2RE%f!2K|9zXjEC zdRBCRD&S(Y4_|coL`WGikk#+r#^eKO=}1t_fqTo+?W(d_Z*_TI0T$J)L+nJb{Lr;h zGG+Y{@Ma`X!J~dXUwhjGKL^t5{P!mzx*J=^r$fO;gZ$rBf93-ZD}ivEvQ~Nl8-PbN z8pmp>!q0yj+6{z4ddd4QB*44sc}sPBV*NBkTZ3_ZZpb;%Rc_;9CUg@u5S89UEUeR`o3*s|wUgqFneAnb72_pF5cOUX}cc z&5I&BM-(Tg1|nn5XJkLa07nwR??=x!U~%BFpOvIHf_D{IzT0gHSa`#(6I$ab;2jCLQf z9p3XAows5&-tu*{#X`HWR4ca%t`0luMDQj|${#RCSG>mC5Wz{?b69WI*>)CC(^V#! z+o-DzPut4c@m@SCHJARVWL@PmTgKg0hJj;rV?EXymnpj^9ps<6<@?8Uq^FXgS_+{K zVOF!7)a{R!!M%cu_R}L4Rrrpo-E!Qu|AFcP)NwU~Q)~wA*3-P(Y6M!*d{G&Mn*c!Q zXD#l}h8*~8#;qoP4LP|=RX5|eZfO0PZcJFoR!q^KO_`6P)Q&-)>T(0ys`+3c1R&t6 zM&%~IDqroJWm<`Gp!>(p7^wb8lPvQ_>DM!pi-t9{5tEfB^WxpwbC`W~dN2V?u*8an zHay_Gf{`d;9?wFz>U$#@U2Pwow*jSsK(dR^w-=RrA8#e|05GAx(%7dXW9GWu;#4IC zRS?BSG-M_UEib?-!w~Xa*1i@4-I?)$wUCht5;uN?-w;hdKP4?Kc>GAbdvC*eP_8tN zfD6xcA>n~xSB(3V+;9We=-t-(0|{og-iUn)*iW}&69@fbfXz@2kJi>SLv#?_FE2*) z?hS?0L0;8L$B}(m^WCj>a{GNWX9FnY=5~>%7h){t8fhTWAgb;xG1zvmWr4rX)qbab zdw$ReJx!)MNM%t;-)nqQerH(s@ehfY)e>*Dfy9cw$?X_j`+!3+I5xV^!{x~5A$AL{^(6h>_Zi}&M^ zFDHxPUiM>YVT;vtgK=pmbV6M=t!;uBpjODJG=Q53R};dsoZ^>+tV34jx4ld2WFW0T zC(Hf>0Z_(et#c1ecNQr^yd;@vxXN#>m+iWUOE^fmoFAhqywJ;)M7ed>h!fxYys`dL zk5NIjDbZnSp9oj013%0seFJ*n+cH?qK$d=32xE!cEvJF&Sy8X8s~U28e&Widz_ABv z(TtvHy4R%l`Ui$yr|Dx@JHc&=s>$BU=U0z$!li*U{-2ETcaj%o87-$%7VXy>E6rl+ zt|i>mgH;1LVUeS+9Q?Qx%BufiWv)$fA+)#IZ!KkTQiMU!h?}rdn(k|8MKyKK_ecNM zU9W?HIUVnytoBN{{6X{V~LH zWoYD|g}i4#T#uBGSU{#M5`Jpv=F#3bi_;6n&dlNuKbC8-662;JUgS5 zyK%h1VsXyprPIi8h2Bu<$S=KvbDhC9=QA0VkyWF`BxLD5KcKTHJII}7EqAsq;^Wv>852?iZ zr|rxM`)N#u@lC1pS48`m=aRPj6eZ?=ocqZQ-Q)P}X5Vw%6~)SpZ4Go@lMi0GA+~m4 z+7yA8KONeH1`o-n=S|-_+9_P7J5zucAHHInxp)7)7XF#LGSgY+C$WLh{OvnapgT{u z-90nWSNFL?`&He|>8(wgq(hTa+X?Z%VlSW4yCX{q7`Wk5*;TTwy0Y--3#&&G&~}r- zGf`&?d){>J^uaves=|;e`~)`1n&}=-T)>h!sUk`}E|V@$PN)w&H^WW&p1dWFXH4Sg zo0ZS$_yhOt8wwxsa@dZ6CPr!3w}JC#K0SyWNH7Q};XI|9%J&O*@0E@L-`y#|y!L3k zL=o22(a~4GLZS{>f%;c5K+_+vT*{WrX(`sIZAC>y&j| z<|0Ab;MfBh6B<6l;HGizK(~o_p-+PWMg@NLi6>h-42)Y9-DW@+1LZ(+6xC)1S?ICi znQWD_^&PFB>$)y{AON@e`!X8#K)aYnzhu;%k8XSG&v(uiYJBKseNIQ9&QPq4SlWL+Lp zOXw1#V=Xm1Pm+%+J|ADMGq1imzO6rANQmig=8-ak8FwK;_O#4C2c%cSgILH|c@wi8 zAR{e~Cy1P8YP`vPB2w+6EpicX8(Hx_27tr1>ZNA;t0PvDo8eLnWC+1$_sg1x6#qcO zC_i)g`xJdQCw%2FvGHwcb2JC&NP|cBmTV!Z1P7|d+7baMm<#AhIimeAF>Vc;)*$gP z!JbHKiOR&vYns(<=lNartBZAMWATbihz^agc?>Db1{Na{&@y)J+ z7EGXJ8noKcuvz9BW8YHgXyH@(M$1meSKnO-CvQfy+)EoyhHY)+UzgvS1M&m#Avx8ocBMk-~~V%5!~26B@(FWa1^`CzBNDoaZjPt z1ga>O*_>3&Uj$IMiekG6>nTP1KZ{D-&Af7iBA)jWlH?$$T_TU7 z)V?Q?^Xhs$0%1m@8yy8 zuDqG{`yeuz_ixb19;;W7?6&qMcc*qgi^B9dSbjt5gAvEF<+oiDJ5*KS;c?JeHGBzg zt$+`won&HoUM{D-6vVKrQH6yE?T0e5H{WUKYD;}jo8JufX7^YC55bv3 z9;EXbbu*jJ)I#SxHIoFE^Qr`zK2#}kxv26GSPfN_X)mECS{sdSw}({tl#^EM>OHAj z`r3J|b@>MD%7HXm(EDI7!7d_F%juzoH+x$3+C=kudUD>?-g{zIt6Tr(PK`5KDq88p zPXj1y8BrvYZe7RhJ|9nBbl_di5 z6X{kcwp3Tb+{UTRt3u(`IDhHf2Js>!H9ip| ztcwppDqHKXnxEw@E10LLvy+v$HI^{d_>3NRr)0K|r_;!}Cz(j|;5Ue=O<(y*Uu(#_ z4!mQoG@FX%@B69O`Y0#`D{noVgZS;#c;W7tBZA5IYFfK3mR@K zT9%n~+9^8^^`}}B6@OxEZ(SLPfMtq2))(0VYxwA_~To z2vbQ!Rx=b+{C+|o%35tw%BwEe_wW-*>{n~Pl+V^*27X10Iq-6emkP7?g|!6EE(kAub%lnp5YsBClop zsy1|!vd0xpJfcAk%75Ne=I_es5UQ*4HQN!5tQibr`gOT_!k-3nzKjo03X(E66=E;? z85FM(Cg3O77Okkm6&s%nL3>c;b11_Js~?TvIPeBr0)#0e4u6Z>J>4N~JJQ+?s$H5J zx};SiF46S1AF9pNJ!>(b5|6>N=7->(=JVW7vIG@`M8jqJ-3S5AiIy`(@a-Yiyv8@|F|^Ib6z_ zFs>MP>6-7Q1GG)TRy3KupHoa?%YMU-hy)ELYDodtPT(G=@hu0pU>t3nOIdHlr=WIu~x9LJt>TV%-Y1TsGnUOR(6e7gVl6TszD@pv-5JWnzIr1AOQ_VQ?D=Q55Gx# zw*h5a^Ix_wUNx)8e&O4y;4sm%VR0Q?iX05qw00!vF3+0133nptz*b2Qv@rG!<;tn9 zWaw5)y|MZH3aiPd&_bnf_hkKuk@Fn+YYkFA^ho&28qJkRZuJJ+-2A<^xAyGEtWWL_ zl|NJz>{<`D;p;ST2)O7``3`P=HyTc~`%xOwz@iV>>rYD2t8(+r)0XKErOqi%Djc_x z`R)7JiD=2`h=9N4C^iyvfQR#3r(9ATOuP@@>3$R|%_}h?Nn(wm;N;`V&$X2AOsZ>m zJLF>de&OZ-OxxM>LReO6fQKUvR==3?d?Tx>(E1aLj%L{JE1wYtu&4*M9D)^4@vMV`wxs#@xyOXnVU)we~toAh%F07yt zZ`5kr=$iQw6SNS{!LNRKULl}wem|T;g2eM(mfOPq!<%fP;&XHT`^$yV#a&@5Wu>Ch z%@lBT%L<-^s6UJ1=S8xr32(dKK8&_|f}R&V(^s#t@n-d$jvuo8dWg>?JwGL*vL{!x zI!x7lQ%`JuXWvR16;NGXll&OE2^6sk%u^Z{J`MvLuo+_2y3!3w8u>{)S|i$!_VaoB zAc;_9tK+Q|Y@zL;&BA?((>GlTXFJnFd!1(zFZ72<76-HxwRl<|OV3wLlH;B3Nww@n zkYpcE^Z$@KiGNYP=E1VJ;*trP`}nE!HO19ie~skIzRHwT*biPwc02Z6a~7Q`c>3UH zr8H(HjCNQ=*kT;{MWyUpbzBhTJcG_u+k2Rg&28roQ-?S`EhGKNKmOybng{=)qlzjI z#uKk|k|4o*HuuEkw!@E1|LDLw9;QmqlA3e?*FA}Rs8H>YHV@{>Y797(idh{#dez{5 zsrU1@L93Qq9CEQjSC-!vhO;#Y4rsaLPdWI0yhc81(0xX*=yKx96-iW^V?2ii;$t#7WqOv^ioPB@O27-7f?Ux>(N8%OrK*zkrvo z?iRT;`0kVJFSn)?oC=v23E9AKAT{>UkCI+k?R9*MJ5c2Hyt-ibhuyupTRLG?rIld* zsOnTuvO{Rt_T~>?zGi*9`ya~WVSNt8{S-D`>b-WGm@>bn9NR`dy`#y9=S=Q&b3VMw z=iIm(-689DsT+kqI%}zmSIPD`?apE_XcXq%aWm*SR_9YPT;u`^BwU^holD0hybA4O)^g zkKT@7>q80BCe_`quTi!^RfP774S6|?WSu;zk|l0`m-z#oCrQ)J^bZb28ZgqUc$#GC zD);4^iUNtZ$~Wf54gFbn8{@bP-f^C=+)4~@k4uBZ1yyVE*APq|qj0xH{|>|tq3p?3 zS#J;dAr?Zt`@Pe?HPYXr-CQ63nG`72%_Nc#Q(rM1q-E)45}pK4(L1%u*D6}QD}TD+ zJNeiR^j?LLL-AFc8<$enI`$qC`FS8WM1Ww2RYiAcC%F4+860A^F}gB&m>)lvn5Jqz zV;zgbrBI<-sRk$>|Gn3NsZ;%LBfr^P;;_@ZN2M7;kIiS@TB&v{Cll974kYs3rWL-m zvAQ4Nk+0CfxsBCYiKfM_@`zq!8xHlZ`viASC;AjxtC{)6OuWKKDwK5EuWHbCEgUs& zi{p4Wb&i=dv485Nq|phfTCXKA!<%Klmhw77zwT7SZ7N2-BO9dcyOX z|EIp}={xADb0lD_juYC*&%z;{Usa?EE{(FUcP>=OiNVK8SFPDb6f#r}Um0#&mM;;I zkL4%Y_61&VgZ3%R9jC_X16xJE#ve?Y+F~u-2=&5&4odu=wh6fXtkSuZxId9!b+cRS zB%cgl({jShyW_{n5@PXpqbj`eKoiwvFSg9Jt#kZsJpv)oKZ(G>&bU#XUhsmfs=durJxBGSTY zw7uMMXW7Nu*|Z-^aOg%I2e#nTI+RqcR;IwN=8TeGNFZuDm{;H<@>`sxbGe^y<1Z z^^x$pb%^!_nfIkV0NnO@)GlvyH+RzOKMn!^sbj;dFn9>8nbttq{%jb@WffpW)3u>inb$4ETRf z$D)85TDCK&wo`hZ8a2N(1F+v0593ERJ<$i=0#|i3Ww=H?7C>a82T((>)U!eeHF#hoj(XaD!$o}s=b>@?I~A9r;rlR}_D-*3 zP7RK%t5gSkq->A#!7Bp{z>TweAZ|4H@2&q`n8`6PmX`1vU=jFMQ0;5_ zdF5!!XSt4+~4zL5YSE zlMb|cuDe+qV`JF762QhXjFjgXQM3+VL7(o+xT;PtQE6S2vVKo)u1Fp(cX#vATiN#8 zZ|gR9#F)w(=6Sgpw_AR&SQZw?NlK6|8)0&p;} zO62dZUZaSI!;~>_^so#ffmFbfn9e|RAK(luBVt$yRJ0- zf7$hLko5cR7fUUUSkTdLzmP7hJ@+x*$amjLC;MtOD!o;v?@wKXYi{th1nVrEEF?qz zkeGBVxSDlk{blzwZp)YGj>aLO$NtCSooe5@E?)@Aq(S+Ae&8kwPoEjDGdFX!fuaMU z)+=dDl(Nt(3-DBPvDM%D5EYbAjmC?;QU037)nWiMu|Q^{v}|MQC&%;d?=3Kqog|Lp_jy8W(lMC@t>`l zpl{0o|9|q86%zK}W6*P0rFL~H8>#H!?&tJa=joCUp;eIEq)Us4- zN6r%cjp@t=aAwhEF{*42I7LhZ?T&Z0UU)Wob@ldQaV^$KPbI5p^p~j%dUY*QCII{C z{ho1Kw_#>ROisl`&2D0iiKjHg>9X^SNdK|8f5Y5Z&X%Pkf$pQl&NJP*b`gifLqXb? z4!PaLC;CqU@{?*s=!BqIQJ4q=sw--@!xZAUIiJIafLEiii4f8u;m4GVm@^-sn02_v zyp>lW^QpbBVV~tmPc!$i96+nKPJr_;^7;nxJet?blg-$+TC5`edgvl8C>LCxgc4nm zL{Ahb)kN|`a~K`t=R&sA06Lp_61mvEyjf=)c~kn!qRVU1l-D1cZ#oMVCcZrLxxnnC z>q$?H5t&HKwmAE(H=)QYaVJa&vVXI7Q+@Hx^T}BnKwHwkwk7bdZDIMi-*b|p)BK4* zupAcjS(~`Cxa4V+E}&QD^c@S!Ue7zv!y^bv>`Ep;(rTwP3*`AjiGrgAwgUQqwq|n8 zReze&9?PP49TcWsX{L%=bPuRLFnGeP^<3$!Km}kg?ath9s}jQ0uw_JYj?FUGGpY(4 zbEy)G7J_-_IzXO9^(#&a_<&Z4I}(=^`!{pxrJhF|?@9)3CQ>m$X&sYJl(@4%bh=(I z74tQzPEhOFM7x+g(Iw^9VB+KirJ5yrFY0&zlE#F}^g_fx09cW9XCde90D3zB*c=G}ihIoknn3ikeG_wgN~LYz8Ms39S>PbWk8VS}cRixKO&aFwWj??5wm=17 z@>Fz88|*FS$k!QAr`^7D@I&A+*N3)vcpXli3y!~+Rz+m%+jEVP^k|7(Xzl|FZ_Rq+bebb%iPViFJx5)zV^pfwI< zf*aL2>WOx$*S-cCyb16JfL4;3(?92Xq@Zdj{d?!25^*m3G4I%1smtN`pVOUb!pjSh zU!I5aRil8G&QehD0Qw9ZYWsxl&kRPe>YjRaN7la`XgFTys)Kn28z1znn!4upCXFcR zeuS}}Jtx)Q7rx34=J z#%M)ys6PB{e(CXggfr$uUc7*0-5BI5S9W^wlWh|j=0QTaYuy0eU&H+lb-wu0pVRzdoQBP|oY1Fy_O(iCKr9ar|Ie zo|LkYs0VH!Oh(vey@XeCa)d%wo?ztE9NoOt1k{UihbsFmAj`nj%r|Xb1W#2cB7$ZG z9kVEco@RQlww~Ylh#6=fa8paL4f?eE9{-l3-*)SCwW<0-tY;@Fu0KFEvMU^{im zEvdz*VHi|n!b<1d>_UY$RB(jq*$s8Y!QtCeDhV zyj%%3HuLfQF!AUTh7pfu;7uH(Yx(?>c|WCQ1)2%t`O z{)}M_tka(rA8@iLo{L}nv}9F2bPxBNc#h?Bx;yYnnM#TP`;1n(wkN2zLx*g9XQJgl zwEzOr<}2)H`cOME_dcH{lh`Tx(HgBp=UFPV4pBrJl}{ho-mq`iN!ICnhq-2dfvT%D z*u>nd_`Hq>BH7zHVCHCZ?@%NNh$B^DE?$1x!Krg+w;X77JU1%Wr=oI~@mG-BStUGk ziZmPlw@3ukG`{>N$-8@68s5Gk%pNjNLeJAp5$a`&Rd{Q-rjSL)LXAB`l3(E*k72v4 zupnsAsv-J!U)pQ|S%Cq90HChmsjnkT_Ormok-dr9c!7UN0o0O zQ8v~QPb#`_Hx~;NcvUFdyWA%QA*BjQqDZ79I&Ll%{A%NRT*TTowb|qr<@-~<()oP1 zPIRL0FNeHRIo-Az4pqBrWUX2$&yoydfp~QP*2Ob5)1FJ!uiE1T6vgdn;I>^2Nj5wn zN(>x4^+qaAce&B;1t;ORp!lA>+OiqJz!1ms`G_?kuv~i4Hn)zZ2(ny3xfk#`yBmq_ zE2u}PW+%d4=u3fbPBG;huGhm?2tT*YnxfJEp30Yy!u|w$@96@}Bd~&j@*Ws?CpuIU z7X{zwBtDvD^5khcrK}#4d$#_x7h2iXpU%;4)y>ReHg@|g#Lz41WNdu`h(_h7E96vs z@ZtuPHaariG=#KEUoejNCAd_XsOS!LGv=3js{Ur@_FD^BuiqR5J^UuHYxK;u$nzz@ zERkfC#}gj?_ecnC6caiy8PriT5OzVUMa+a9w50ubpCA7`i~8~er&pv2hoQxDV7;+` z$y4=73H$0~vpOmZM2Popmz&KwXYR8qiXpfu18Qwvcl^GYS6?TWX|P32$C>DY72iN4 zYKYJnvFW!}5+DC^QF?p$jY%gVwg!a!$eHo-?b6uW9MD9KA8SH;@ox6NLgwyQRbjto zGCW?+^zAFl^PND8+uCp7Wl*q0yyOapK79bW@7 zs`ir)|60;NWlR7I%kahM(s9{dXS-1-Jru0PM$#929GTYko`a0Jaxa(Em>AA*OC{HH zhkXq_{^DZ6N4m#YTAJS3lt{(FhDK7cmlry3!Ra9wCRykK^^hyza-( zbaZt3Yko~Be*u_Q@Pwet_pPFGcflpvTnn%IX;~I^9j#k>?d0O`lRhG1K7V0uTCotU z(#1TLdPy8=iGSuc@IkGWl;&$2*Wd`gLm)4<&I!tUJW89#sbfjx0TbXt+tHUW*J9fa zDJ7-j#7#%la$VIRHQ2yV!hsp!%&f zl3Uh5Zrf$?X;g*9o$J3>!(&{>uQhf;d>bqENBn&%V4u}Hd-c^kOn;?X&-wfgX}}ZH z9+qJYr|eyf<$8t21*~TKMJE6@NIv}Wz0P6dc||pVsddu<-tlw}{3HjvKK_9QT>sgTofEwf@a ziv0Xptut|hr}D(Uro(+61s^qrqYOe{`d@>P)7ah36f%k{r{kD9H?!=6jurVV`yVY# zq)ZmSO%)raBy0*Z_#4)j_zgvzS>R!e2FLB^bq;f3K=iWKAw{H)!t;BU_cH9+PGl&e zo}Ayckeh`^lR`B9oMczfCvp{fm9b#-5z=&}c9`{~U5T89BKWoAyr`+D!UL?)=Bg|r zU=s<9s|iPm)%R5{LaKC9!Nug|UnO2>l?8|r?5u;A@4~TE5LAi8D)VHZr{yx)92kY`BHKGI5K<`HVe;4jG zI3MHe8#K}@Cl$TNq!ZzPGgR&k6jWGvCpk%ZcuKWkxs#n;MMHwJM(w}el+98-;^9Z3 z1XzrpFtC6#=mBJvIhD9dStosC3q56Rh3&XH^ostRPU1r%Iz+&n-$3EC>!>>A3317v zmCjF~%P_i-A6Ms6L~H8R>HX`p3x%&Zd=J~?`4&A6WIYr-jwq5OSDx*Jf8F9AoB6aF z)5SXO_Blg%DmAB-SlBJHUp)FtRB!~8n$xJ+@ZfVl@!^2y;Jnb@Qm|~_*!pfh;^UZ2 z(omjt@WMp_g4}ApyCXZ&%9{{Jw+gLHlA-hLJ-LEs6y4|;nzup9ZBfq^|i(KDw%quQUSaFqrFC|%+o&U_rCU@KCzLaL#LOfgf zAh*`_r_b}Q6W%3Pyogo6DSnAN&_Hwfw76U@@Jgbc=X=f)aL}b-Q8SPeuBN@)X%=! zNWZFhVH^HsYI&kkUm@-#8%-Q>98h#Y=HqeC&{m>!=~Bd|qT5$Zj-4ZufYqt0g*(eM zoWPs44wetJnJFVb{PEp)iV?`L4Iv6%Wz$Z(GQ-p3_Ui=j{iE@ZQ0pE_WD^t z+=n0m+>;?oRaK-^-YUQC31*n`QO`9Q2jYoTm6YW-#X;mNFYu7F1hh+{!Qci z#DbCym6gjJ^91jIQ?uC$(cc_9p?$uQzHOY@lZqS|od-QLFlK)qGUm2F=)~zd2A_xeAAT8aE z($do1NT+mnx1@s7UDDm%2-4jkN_RJBzUcbax4wPOzOMc6{Mi4L_vM-6nRCoJ$GFFR z54tam4m;f0UH5T;1AkArKj`nVU&AzZv`7)R6%wWZ95&OSmZ1mH6pC$xD4E=))QY)& z>>)-Oo85~3k^-=%Xt6Q$D&b-IYzCXe!F!tCqeE$?fp)4dXW7iUD-K3GT;7SLio?#) zX*qi5+&z0%K}E5x@2~BC(T4FokwsoV_ns#Ky+=s@j8;=&;MmSKlb(4fQ2Z6WP`Q%i zNm*w|Sp=|ax?A&G60s&1x-O{!&7|2if`-7C)+T+*r0RbU^{;eTiMEx_-zu(-%CyGA ztnP%Y>7H@))1zm-7AGxCG73l)%E908;JIrupv9EJr_G))xuw)F3`FqOoUQpCq5k+4 zC30SH^$Kg38|RCXlCG-Kqls$zB!SqNgdVvV<`>mFgqqJNG$d~KW@}=xXp$J8Hd|^E zckBj-Ok{RR4J31QA46&oE{xyI@jrT98!QgL)H@$FJSp`UmT0ilRwp!(-riKC(v^=T zc&}Xia_9w>N>mGOi7_j`vU`fj*SBsci2H9+=ypbLl2w}y;9u7bpl;bydedQ8!Mo2R zAiB@%-0ZRwQp0SGJ}=VE6nPMyQcJQG?_ja`I0QN;Ro~iG^=O=2RLd}R@YGDr&w&^$*=MP`bv`p>gWR&1IA+oX{cd(>3V__2iK}c zLe5q%qABr9jFsvRmqxZN<6l``XjI&2%}NZ?&?ZUgo)`Gv67sz+w>~@7vclW_uoa`2 zbvbcUqqMhu(!6&~+nb*gG#<^zt^ZSdtUw*{%NJNd=N2RB&7NnLS}u@u7>=wyiRP#Wu4pb<$E>@BEb~^FXGM~p-;dM zKKs>bqENgDv*H_*NsU6+f1B+b#Xp&DawQc9_wka`GM>Pz9}!x}(^sU<}Vf69!&&r}ezdhVQ-(IXmSfr1qof=)=V|y|m z5FwP@V8Z->`6$6fr8ueZGHIvW1%A_GWQL2*iH;i#J$rdR13M~|edM~A%M^t4v_MqDLS{`5B+w=}J^0og%97K zc8KDcwbR&8CdpVqt1k0BL|j6+$Nw!&16JGyJG@f4x5!Of$pGjDX3~H0mZmQH;)RJY z8+Hu!Pg5Ix2WHpf0<$D4CEgIpj#JLzA7*y9c@7uDX>NROH{dUsPSl;B&9ti=-*O+B zorNOr?rH~Gnto;K%^f_C$C2IVN^TmgnLunq2jFAR=li^B`*&XTeV9NC6NaW`ZSZu6{UNKkijo~kpH_G(Tt&HSV7cKgPtT@fpI*UZVPI<~iJbF=)Cg)Sh zqL;LNihs=7Si-9+wQgGqkT%BoicWLA!*YS=aJj*+?NCA$)HqKmS0C^NCe@5Q?W&e3{uO z`0t5~eWL^zS{(VoM;;P{Yv?i@vx-u}48emU_?@k5KdI+}oDzrx+1tvqJ8dRJVNDM# zp&vX??$q1-7PH|Z9c5hq)j(E(KK*iD%GMKbv1J2C4AaH0m`58B2Umt6R8Ut7%d2#l z7cmrs+nNBq#cwv?Nm->NRdFmz-6Vfk<3*K!J=0fh#e2%Hw`ZbwE=@q&Hng$0FMuA8jyI#-Nw3qoRqcVH4NwO0)3SMN( zuD89;#v|#$!U@OlIJqs7sRs0I#x!Tul9UoqF>wQ6 zy%9T=U7xJ`2al2)_Gl{4a`Bo%Do3n|VU2y>1XV#zqbwB6aZDNZ*b11giY+YH4TcDL z6Xsr}P$7b%RvQcoCA-WHXmw8r$jd&kJ~_c3A4_qW8hh}J;_LguNT$q(EC`EH?GqVN z%>8j@$49DER(%E;V0BLv8$0UA;5M%=&F7M4Ox2~hvKU9sI$_zb_7e+;K;S=Nwfc?) zU4~wd+ua8}yYBU7SjcwmU~Hg2b#%-EFGA18WS-7Utd?J{ zG4p@XQIm;1KMcB^NByx9PvrW}{K$cOZTq1^ac1c6c5T!6<3CC+XBs z;v_H?nl#a2?`^`>E0qQKzBo=Hw3m|gXWSrEjfCFWR67O`FObxb1#FG#M_RxRDTsF>Cf*L z21|>3;HpbX-WpwPtmMRU%|MHw;E(67Rf6-a!t6?!o3nLvQC`o%{=-VKW|I#{9U-Je ztxwb?rapD1vvd-;?4oXCdFJBZ)mX$$4Qrdu0w=K{&`j5I{J2u?+(`hc;`9WJOS9nhXU&tfJ)!>C}sLZdA@vVgU883 zTaAF_?N;87n4gcgDYtT!PeXMYg^DeDKM`y*qGQ4$nXaVm?C!61fkH*)!Bm;S4$RA! z8B6PI(CUu49{93@Mrfobc$8T!CO4l5rH^;7ojK$|7^i^dCh(R1gQ+ueq#>+Pv`%7V zEov#E+}*0a^YO^Y{t$6Hd5SLkgS%Y9&V3KLSnuHs*rSDEsGb7= zUgPlA;`ky+>pH%8hV(7IKZwR)b%NHANX+gPwc?xTc`3?5Nze^@T(5mZJi}w>qpZC?lOXN+276nW0Jv1E zkD=o2Xj-{Fe{>q~7}Uyd18;v!hhT9VBfX8J(=2<9Sr8wghFm$)KU<|!k9uk%5zib^=3}p+f$P)YQ(`?%)>s@)6n7-|#pYsbhk~BvF@)~m!`c@n25b86idH^% z+ZkQt1Ad=EQT<@ex1I#$lLzE@U{R52d3{ZLjVV7W7ZQtZi!M3&DwpF?=iF?m8sZ-%5>m`5mE<1|Ez)y&QQ)VINANu4vq$n& zqea}p8Q&z0N$#Qo^4#z!Z+oaiW6}?QRbddi(FpY8m=LCBmWXDR!PlP$LF}3BHitPh zaYAMgEU~yVkuT6zVPyin+wLY*qLEu}@+dFA$56~YFSMB{(T%FMniJ;F95UwVC zuwv_CZ>Z*Y?(rph)LRO@WhLdS&7)k;gUARo`Et|Tz=JAI%O$kimq{kzIwkY`Sunab zH^Pc-kRd;@KXFPbb2Kmh;ORl0$0X-o_KxHHpUkVqK$CXqE1`VV0rc$>^mODC00Nc0Hh?f<*yJ4#}EyZhgJrqJbzM>xqv^O$+J%~ENxf;D^&B}H^W?OQQU(LQZ9Xzp8rZ5&<@igo zZ^*+E{h(b@dS~QENk9ZNP%^;@0}gZ8Ewj$zsZ~JmFC89M!DEmN%6p<#-A!?KK_Sn3II_!1BTJmYRxi`%l zct(I{AZ&z*2&ft%469QDGAU~x@ykmbwk)jS+?fj&1-o0^SKWy6DZb2WlA}3$7{?$?;_1#;^^jTp(KV ze+-ZvHK-snAB3Mw4>_A@yx}?(G3r7;>J%cZRlxZ({v^iv*_Y8By`qeKoNd*()NIGy* zdkXH)M4dtucg=Pf8+r(c7|wkUBEGa0J1J3mZg^tvIUixVJk&$fZ{yD7_RCOZRQL}@ z&i8}+BoJ8@GZZD?J$AGGBX5LuVn)chTVl5g5;i+aKS3E2XFi%wr3+>QkJ*!_1+{^d zJROpzpwOF*f#TjctbG6cc`bRRY8G|14J6aUqw9~*`;MC-v9Dy)7f(C<^JR;AMRG-6 zyBZTb6h=l(Ide=1uxRhjepNsfmPuMG&}DKgj!p@NqIiCs z$xrCbq_=#Bgn{@=zz};CyHtU{M5&sH++x%A7~_7vDx9ByG6*Fk9yq6iFtWhriHVK% z11y*lyQaK|qY}n;2A3bk)1E=1*39IH6C+6PIU?%MV30s>jp)k;t%TQqaRD;EAH+qu zm)aglh3QsRF<=XnOuQ1FaT`Cbb>X%8!32sY7e{N8-MGcy$UyXJgzn^DJzgDXFNIl< zg+1nH8x%+n>J&5{Jo2mOXqK<8OU-?S8P#z-^d3!cH{nGz(aGzFtj@TN9bcT3lP|ND zY zDjA$@gT@lmHO_%+08_V!Btl1CKd;HN^2&KwFwSMy+k|3lZuqxX-smx^^1|&tDbUBQ z-%OD}RkhOlfipXLk3S5*I%cObMnT874D?USo~sy2njuUa2quP(Zn{jE8li$c{yi*L zc}7qKPN*SGt|^9_>MLllB_#&3(N%JaxUN`9L9zg@KaZ+m*$Slu&JHFNEo*WywC$%J zy*~>6$uQJ@mD3!FD(q_NQG&OB%LBwYpeqU)idwyifY61ly9SXHFmQ3!V`q$z7 zypy>-mT6QMWJVAxIyuOCF;$<27&iI5W>_XRX+J<$|h0iz{H58@-=Rf6C@}neIe32=%8_ zG(LE+`lQ66)h?P+@!<&m657%KU6kP0mjN099)>&@45JvUU)Di=lkV)=?8A^b|A#nG5ZzX;+K>89vGzgJ0z5JbU9azQOeU>bl=u1J9N{} zs0HIfE1|t*I4_JOw{jycs#(oG@(#MbYZ?=f{c*q|!kq;H z%f&QbnN6U2r$JH3=J^eCF*u-@o;1xh!X3ADY@q9y#Fa)K93RL_;O}^YdgupRd6Zqc zMh-olA%2>;>SvwenB6{#N3wlsU!*jEvIxHVsYQ+3@&}ci+@G~X&Nmh=hF*V zI?b}xP0IeXHiI-WxF&f7U1D-e!>A}fZDY=6ccY;e;6As_K=kQvXyf@{Zq9LxyGY2V z$rQHyh?5Y*A2**m!yLHu&PtT_`nc0QU-_Naq-z700tB#jFUHZ5vmpfQT?5-}UG2f} z{rE;Kk0a$&yKlNvwYtbJ*vE!~`wE+PGpeKDrL=0OCQPWDi~0#~l0O=zF-N+MSkRK zF+CrAUPbX?0Hg3tQXTrIPo;oJ!}US5jRc-khx{dvXrk-dym69MzEN@??Pt3AOal|~ z?>8Ozq>uA7;hca_rg^lu zd`rCnL91@MPM$u)4EmHMZn*x5g*u+>Oc=1Y%F*zLezA{Uhq?oI5xotCq-e(E9w@`)!L4;o^jnoA#)4OGTXu#9pZanewxxSnj5Abx1Yz@SjgEXx|GV(?tc5`<91|rL$(IMU z@8}gz?Sy?jc=ps9vHA=p(B$LY7M^N7aS&ctcn-nIXol6J%ujAd>+}o7cSNtuMkif4 z>eK%u2VJ!z0%|W7ELSYXjK?H4Xs0wWL8QuosB@(B0^zNYXz$F8!Htidd*)|BBaqOf zm-_a6p;PBr0XkvT;h(R92L-O|CN5nKZx1E0UYUo-PM-8lV7)3pNhG^m+3c^6Bz7^K zdDD1zT<6a`1Vo`6E77OZ*_{ANYk_0~*ca??dYPy-4TrFi=zA0)dq{D4wVeq{=#fnG zFV++c5m@zc+c;dJiJ`D@X1_fV(w2=S+eA;yl!1y0x2md8eG^M#rfAh#^IYv4t}iae z%`LmldQ4n1zk3S!{IES-39KK+vZt`sX|{!>K4+uzX@gamUWn1`2r1hN6Z?whUsex) zTtzSD#ndg|^^M$m{jPgdf_dsmSfk|anH=-?Z%Rsr4mg{f#YW%~W4XV?aPOI*nDH66 z_Gz~KPcd=H96p~~u?66OH0viy@V98j8;|MZ^q!-f!Da4N_$!Zjat%l0pAXQ2``l!N zw`6Z?&*;fsOSbqkKkt~^tjt^4?95L;KE3{YG?XcXUdT2^{?*8Ji<(6~#oY20!S(5= zr7q%OaBtMv_M*ILunn4Px@NQY=gQAg+z2M~@tvlq>v$xmb}#l|jFgR%%+`@YpJPGD zn!>H|zl628Jp9<{!fWiVOA6#>E?*l5ftdbdEZ5j50co#Wq{Y?WWNP&j<|+?#*_j5x zX&be7?rwLM-|=5P1H>dWTWWU=9LFLJ)U0aqz(SEE!@`OgnUvJzoxO2kVi?Km-s+lwV%t}G_o5ZkSmnY)XrmF76Sv<-Gn&{}O#e#8SigrAl^bC05NJx;P zvB@Vt#$7^hfB|q70?Y{hK$o^IFz_}q7WW4;{;$3-Qfmt0$>4?F7uhc#*J}I=ZgGX(+QZD^sBUFHUR$VJC?$Rv6 z>a6{YcL#$fQo1QhSl6k!NKHI?bT6)F(o#F4+xAV*rzaqbeCjG?Z|ygEzI=_FdPKVf zD+ak{C_kA6BK)L#I3l6#UhW|uB)eza>o@S#%<&54v996D%Utg4%HZbBmv$QIRPyXg zW;2J&Bz|x#l_EKZND*d{S2)lnXYec>`Y%$0wf3y=%wH#Gr^O?`DdRtO5Xn2B-wwrat zR^oC#YYUAsuYu|1cT{;z#lG>52aFQht;lea9`cIM9tR4rI|=rE~oxJq5;nPlNyH%BDD zYLpkHdtiY&E;C80!}?y(s+I%avMy<4baW7B488V|&zr=#Gt<0gI$O*To;e#29&%E$ z4YS!HLdQ!M-?D{|bJK zY)MidCYk#h$PtuBrcy3P4y?W>Ecub&El8iV1JTC%(BxCat%*SMr>-BZlU~X+J6)qb zk09t8*76k0J^WnpMM#=a>~t;Ewxg1{dw9EV0BfiDfl^i7oy7CJj)L?w;1hp>jR-Xc`NY%U4{%eWLs?B$D)U6 z*3LrftWtC{jdH3~aM!(Gy70iC52$;CL<7N=ge78q@m(}dv05!I%>p+(2?@y$#h<2g zp8Hq?O6PC%JU5|52DO(|=&x@E+sBR|nvq2K6G6SecnY-+sr%K5W|z)tsJ>}HAyA^p zucWDSDabMyIiU@<2)$}&vxVf%wwBu$KP6gFABuel;_KfA9bYgY5FPhj$I?){%13rd zgBEzj^48}B&-@Dgsa9gUd|w4yA$WT4i^RL~jV2!0Z!?swdipR1Pk>Sa;dZa7PQo>T zY&$0F4z1vYIlIF+tf`uSgEUug`>ML7AF4PHr@L9mATLEMl~UBY#PoOuN*{FsPf7d} zuFdBGC-rhaDA$_yx$qlLxrV6|;yNw}nrVXb{1rW^Nv&^hw_vvGp8KZUg-i7-ke4=F zFd~js3`XpWSfB5B%plp6l^>7lo@Z7VP`xTUdr61jd5aSQjiPJ)Q?>xLSTr^J`+fi^ zV$fZo1_i-wxY(Nn0{jiTlV~?~{+uvpCXlWgLbNpg0^jaA*Hap9Zd|JugWY+1O2J9) zhOQZ|i@RmFEgAXw;JS!bG&eO0*@`JPo&VyeF=z(4c3-<(+u^sOId1 zrEg3^H3RedEZ@tF**1RCrTu@Ez`v`T-07~x5vAi+eQF0;epy~2;`H*aqN0PEp|4mx zO5$rgz`vNu&i1&Stn7RAf>SUWN*eo5al6d?t8|0?0S$m8-a-taDGxQgjB_>{q$}7z zi8K(e3ryr$pghWQ{~qXc{v)4gK=^{Y@Rf|_S7zWR_QG9=p>GK3c<|*7{dJ|LICR`a zhn~Im811Cs)dEO9WqzF%k$PgYjQ}+)nhi8}cZ*u!nirz7<%i@^1voAPGj+@$3%nWQ zdHM_lAkXHD&_N%4)Q{a?cx0_;rnyZI<9x!`Y~*lVMzGQ-pg;YoXtFYseWwznLP#m8 zE@^$^xdd+H?56#z6F6w}`VX1F$wCW_CanXMvc%q>%|E1>Py6uKxM3y)iFRxLm~_RC z=^1!L3{mocEgZ-6nNr?m=gE$H)EZm0$fVeWQw&mK?oH1*$$84Rxgv#BU0E~lw!K_T}6Lb;y;qHUaFwrFe7naI*3F{qn_^G3S?MDcg}gL@V*Xr7!xfZ z(<7N1jN=hGF3(OAtYSBiw$v=g2C28Vn4b|F6MBLC#JhUCCyTJ2G0PX@JF^&70=(T# zSZod|?}e+YQpNTjUa)Y|^iVkRB@kTKGC>fUfDB5Ni?abqm@sS_K^>}QZkGxE#=eyln4qYf8FeHEq}u} z3q^=?fgS-6$i5{njW_#fwa25!^EHL_329kBr7Oqph9NMY9}Sv;ekM5uoQ@Z8+3@uJ z``4SNJBZKQ07H|}`TNpXsD(M-=LOh9lNHA%;abF*>!rasFNmVjjR4g0hhpArF@1St z_MV=}gi`qz#p<%MeXS%Id%y-c7&-fyGCvv9;;#YaXM4##I z$8W{)R*S)I$xE+T9bBj%=jtY@;j7QHUAUc^aLt7`MA9ndOkpYJqiVc=E_QO|p#KnV z&-DZ6M}*4&aq={F=GJ&zLfQG|Qtr1hbQB_$Igd@hwM-jXuOIfIs~X7+u6q2%nqLfM z-ZH>m6$LK#ljA;!u(=N+o~*8$U(Rznw0FDLp(@R~4Z|()X{P_I`Xm7UjM|KNyx%YM zKxyr(F1*(c>sT)}g@Tet!{T3|`8af2I$ah# zNBBrYN4xVb#{=&fwJg#J&ZNLvF=xo(LR=V-!(H27(c>-|r z@fMsCZH^U(b)~R$iLmmHh}OiMLXofjCcnWqI=w&&e?kEP|AiRh{hrzn+W->e*Ws^m zx=kC@D&nr9Q4!B0@Y)vkCYJ%)8U%FA$B9Mb6|g*etPc#Rl*hN$5N>d9tc56)&Ly9& z;r(iUu;X&_>U>-OY@_+idiyNJ`e2Ih+Wtf?o@V1c@)Oxi6$H1r66_hfJ!l}>jN6S% zjD~S3UHYEpUgLJnu)EkqbejNYkmf0q#gWI(@@6NS7D%Ser`v8C6DCm^-lz63-2G;} zsh_I(5|u(_fT(lE*m$}m9A@2Uxhtu;)g(OyRf)6O(!bMNUhtbaXvU!l9CW9lLo|%Xi5Bw>-l!G4mnA+6%|@B-2{HX~AJW`8aNOK<#j_e5Z!dH9udY8Q zn3QEEj1~LKb%vx1>A@pykIP1%O`PId-JC9-?+j+7$}N*HjCziy(i{VXI8C%s&G<-E zV60G9k$hQm20k%15Y{PGI_A{nfCUQ|V0&o5Pmb&%7$54x5zT&*x4ZWoHLZ31PfQN= z%2oU!BVWF1C!kDR90k#6qqzZK^5AR>M%Kc$|74OQWclY?AlDP(-QWm3v}Ley55D4iAs(H(IRES6<}R{(F|^@;*2AQm)(Z*(D%*G z0_wBp1&6u1BEDG7&Iu$x99LDjnwoFODvxUtZQnBgWZo2@>qZ^~w}Tf%e!LH)-=HbL zw0?M*x3YgB|NeK>X*! zA9t;`!jwUXsnd8+3n>Uc-5$#6d}Ok%o(~Q<`xfI8+{d00z(lSxpN+*so&UbwiJl57 z1DU%!F6W>ZYY?IrpFX|S0lCJkB*Ta z8nnn8o;-?qnqI+iR>-#7*!Oq~K=-h}3TAzoVki6Kyp#GcKiEvwrm@(`>AtL@k0Tdz z9PKf`Nyy4tN^o3%xE5!NK?|E=UmfEp2V}jhiwN?>Z>IZ>zL>Xtuk91ewr{T^0P<9d z3X_GF?G)&T0?O^}o*y55!=-;-gLFqW$pxRt

    Mt^{Gfsy#=SZYeA*qjkT7Qx$eHJ zx}E^d092g}W+?whG809=<&t8@cG=(8#p!=AZb`r(b1fbA*XiD7e_-X>dM->9!j-Bt z{YM{C7ys?!5vBZWVG-Vo&lHjI2zM*<#!k4Nt@Lkx_5_uU{u7K~OCW}_-P>)9-Q(r0 z9J9-JC>%1fvK%vCdN9k+Hocv8IGe)d@)$pZTQJvRa~`jZMkRWJZS8=A>G7IWQfJra z1goVp=pSD}UAiUqo8C~BfO`3DdnS9g;*A+d)U_(^>Kw57^a6tphaeOLt35)M>8bcd zzQv^V--!^3n0?Gm$qO<%jW8QU9;VqNF%)6!k-G3!&T}S1q?zdsJVh{L^dzM#nH2w- zCCkB|?MbAr4-l(p z;*)0J>zZd->~MH6oex*T(XyV0;|uPeFQ>z}Egr}Z&3}Uhy;P!H6`hiU0hw&313J2I z-u3dDbd_#C2=q@AN-=)1!)yKl;KT2C$`j>>J{n(*?!hzK?j9;wF92P34JQiA)|waS z8!+UUnX~RI+1_MDgtRq_G2rWxt`R54PU^O8L7PK96QZ?|2iB8Fhv8E4ns2yb*~B1o z6uQ`p=A-9G^mq{g`<$3vK#NQuxeK%_TZKfJ=K+u!pv?cJE5jZxtx6zlpbp;>e^~UN z_=G|YEb7Te?VVq%PjGN>D$Hgd0y&xxQ9VDI!_gETDFn=e5gILjih~z64mDZ=Pz42m zQ(4%?glw-Q^03h(bp)V-UPq}-{R2(5vNbBfi5?QFw?I3JO0*KV@}pEB-N@d8qn_rC zAww{%Df1&;y4P)eL2tLYogUVo`4bE8xq=kIL@ACPx^fH1vj2G#{$bhl(!i4Z(BF z05vp3#WS&qeRB;M1-W;lHJ699!uW3CI8QAlnR^~P3;Ebjeo%6HmK3xSu~M=s{A_@? zkNc($qpimfLMinF?)6v;1rpcwu7-*c5(Bw!vh=aF$c5OiusU8Gqlc=%$F~YHlPX;s zT~1-rF#jSWduUj3~eF{1rxHY0h-&p%%Ggn>=_qiREBq~vQ!2Xzp5v$1T) zVx#xGEy*rhYh9rGDp#6_ost~U1!F@7x%3R@rc5ES9Ukvc7}HHC4{|1`g?sAR95mOO zNW3)Y=<~(_(w}Y_vPPimhmpcVxz>0Wwn-D2=XYB(P$%(YfmsGn8N2;maewh7jdnZ^ zN=t)6!Y~ssODcT)vkWFUQui(7TI(dTsLp%R(kL7?Gjz&vJ=xyLkT!&gfNpA~B+Wul zDrz^%9um(vxh5+hME#iqoxSf-ULcoC?w71$yXF2YU&E+qDF~7|L3D(D5 zecu}AFJmIVJTL;55dYh7fAzBZA6T*hdHpCpT6l=)f1#yeaKHskn1(=a1q&-{r3)cn6r2l$t z@Cn&q@Xvz0k$%|f$u?|D zN`K$YjKRBr#OgSMNVOJk?=O!d6hZy!VJ#BAAC&~^wSPG>LWIF&S_40ifp~9qqlqA<2iPAcuK`O8Ca>0o22P z{R0mw(B8;Y65&68{;w}U{$WMBRMHeLfc5vU{o~_5Pqc%iy{B`-q2Gk700TH=zq8LA78j1L-`@{RzABQ(SJMO8<7^B`Dr!z|K4^Dgx3Yd ztEd70{eb^FS@-kvzms+Ez5nZ3zt58YU$-H{Y!&+h4#~<9a0WgQ6_OGx;@9^1KfEL< AnE(I) From c8263735986837e7636bb7faf8f3cd40d01d3113 Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Thu, 10 Oct 2024 09:52:45 +0800 Subject: [PATCH 180/265] [ISSUE #8798] Fix typo (#8799) --- .../rocketmq/broker/processor/PeekMessageProcessor.java | 4 ++-- .../rocketmq/broker/processor/ReplyMessageProcessor.java | 6 +++--- .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 4 ++-- .../rocketmq/test/client/rmq/RMQBroadCastConsumer.java | 4 ++-- .../rocketmq/test/clientinterface/AbstractMQConsumer.java | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java index 55552003d80..2c0a1cd54a2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java @@ -258,8 +258,8 @@ private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult BrokerMetricsManager.throughputOutTotal.add(getMessageResult.getBufferTotalSize(), attributes); } - for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) { - getMessageResult.addMessage(mapedBuffer); + for (SelectMappedBufferResult mappedBuffer : getMessageTmpResult.getMessageMapedList()) { + getMessageResult.addMessage(mappedBuffer); } } return restNum; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index d3bb048f75d..a70b48debe1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -115,10 +115,10 @@ private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext c response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); log.debug("receive SendReplyMessage request command, {}", request); - final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); - if (this.brokerController.getMessageStore().now() < startTimstamp) { + final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); + if (this.brokerController.getMessageStore().now() < startTimestamp) { response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp))); + response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimestamp))); return response; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 0a45f096235..0e5571eb130 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -1619,10 +1619,10 @@ public void queryMessage( final QueryMessageRequestHeader requestHeader, final long timeoutMillis, final InvokeCallback invokeCallback, - final Boolean isUnqiueKey + final Boolean isUniqueKey ) throws RemotingException, MQBrokerException, InterruptedException { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader); - request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUnqiueKey.toString()); + request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUniqueKey.toString()); this.remotingClient.invokeAsync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis, invokeCallback); } diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java index 2a596197441..7ac5ec39786 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQBroadCastConsumer.java @@ -26,8 +26,8 @@ public class RMQBroadCastConsumer extends RMQNormalConsumer { private static Logger logger = LoggerFactory.getLogger(RMQBroadCastConsumer.class); public RMQBroadCastConsumer(String nsAddr, String topic, String subExpression, - String consumerGroup, AbstractListener listner) { - super(nsAddr, topic, subExpression, consumerGroup, listner); + String consumerGroup, AbstractListener listener) { + super(nsAddr, topic, subExpression, consumerGroup, listener); } @Override diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java index 5681ecc841a..22193bb4ba9 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQConsumer.java @@ -69,8 +69,8 @@ public AbstractListener getListener() { return listener; } - public void setListener(AbstractListener listner) { - this.listener = listner; + public void setListener(AbstractListener listener) { + this.listener = listener; } public String getNsAddr() { From e75554d5a8b7708d5a8a5ae9bd723b614f8adf7c Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Thu, 10 Oct 2024 09:53:06 +0800 Subject: [PATCH 181/265] [ISSUE #8804] clean offset when remove group offset --- .../offset/LmqConsumerOffsetManager.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java index ce70b1a820f..53e9e2e0634 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.broker.offset; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -110,4 +111,25 @@ public ConcurrentHashMap getLmqOffsetTable() { public void setLmqOffsetTable(ConcurrentHashMap lmqOffsetTable) { this.lmqOffsetTable = lmqOffsetTable; } + + @Override + public void removeOffset(String group) { + if (!MixAll.isLmq(group)) { + super.removeOffset(group); + return; + } + Iterator> it = this.lmqOffsetTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + String topicAtGroup = next.getKey(); + if (topicAtGroup.contains(group)) { + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays.length == 2 && group.equals(arrays[1])) { + it.remove(); + removeConsumerOffset(topicAtGroup); + LOG.warn("clean lmq group offset {}", topicAtGroup); + } + } + } + } } From 11f0002a3c85ae449d72f911b03b7b74e275b265 Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Fri, 11 Oct 2024 15:29:18 +0800 Subject: [PATCH 182/265] [ISSUE #8806] fix autoBatch bug when connecting multiple RocketMQ clusters. (#8807) --- .../impl/producer/DefaultMQProducerImpl.java | 2 + .../client/producer/DefaultMQProducer.java | 55 ++++-- .../producer/DefaultMQProducerTest.java | 168 ++++++++++++++++-- 3 files changed, 196 insertions(+), 29 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 74a2516174a..3d4fdbec373 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -250,6 +250,8 @@ public void start(final boolean startFactory) throws MQClientException { this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); + defaultMQProducer.initProduceAccumulator(); + boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); if (!registerOK) { this.serviceState = ServiceState.CREATE_JUST; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index f0842de8ba7..a8bf7cee85f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -174,6 +174,21 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ private int backPressureForAsyncSendSize = 100 * 1024 * 1024; + /** + * Maximum hold time of accumulator. + */ + private int batchMaxDelayMs = -1; + + /** + * Maximum accumulation message body size for a single messageAccumulation. + */ + private long batchMaxBytes = -1; + + /** + * Maximum message body size for produceAccumulator. + */ + private long totalBatchMaxBytes = -1; + private RPCHook rpcHook = null; /** @@ -293,7 +308,6 @@ public DefaultMQProducer(final String producerGroup, RPCHook rpcHook, final List this.enableTrace = enableMsgTrace; this.traceTopic = customizedTraceTopic; defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); - produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); } /** @@ -320,7 +334,6 @@ public DefaultMQProducer(final String namespace, final String producerGroup, RPC this.producerGroup = producerGroup; this.rpcHook = rpcHook; defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); - produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); } /** @@ -1168,10 +1181,10 @@ public int getBatchMaxDelayMs() { } public void batchMaxDelayMs(int holdMs) { - if (this.produceAccumulator == null) { - throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch"); + this.batchMaxDelayMs = holdMs; + if (this.produceAccumulator != null) { + this.produceAccumulator.batchMaxDelayMs(holdMs); } - this.produceAccumulator.batchMaxDelayMs(holdMs); } public long getBatchMaxBytes() { @@ -1182,10 +1195,10 @@ public long getBatchMaxBytes() { } public void batchMaxBytes(long holdSize) { - if (this.produceAccumulator == null) { - throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch"); + this.batchMaxBytes = holdSize; + if (this.produceAccumulator != null) { + this.produceAccumulator.batchMaxBytes(holdSize); } - this.produceAccumulator.batchMaxBytes(holdSize); } public long getTotalBatchMaxBytes() { @@ -1196,10 +1209,10 @@ public long getTotalBatchMaxBytes() { } public void totalBatchMaxBytes(long totalHoldSize) { - if (this.produceAccumulator == null) { - throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch"); + this.totalBatchMaxBytes = totalHoldSize; + if (this.produceAccumulator != null) { + this.produceAccumulator.totalBatchMaxBytes(totalHoldSize); } - this.produceAccumulator.totalBatchMaxBytes(totalHoldSize); } public boolean getAutoBatch() { @@ -1210,9 +1223,6 @@ public boolean getAutoBatch() { } public void setAutoBatch(boolean autoBatch) { - if (this.produceAccumulator == null) { - throw new UnsupportedOperationException("The currently constructed producer does not support autoBatch"); - } this.autoBatch = autoBatch; } @@ -1439,4 +1449,21 @@ public void setCompressType(CompressionType compressType) { public Compressor getCompressor() { return compressor; } + + public void initProduceAccumulator() { + this.produceAccumulator = MQClientManager.getInstance().getOrCreateProduceAccumulator(this); + + if (this.batchMaxDelayMs > -1) { + this.produceAccumulator.batchMaxDelayMs(this.batchMaxDelayMs); + } + + if (this.batchMaxBytes > -1) { + this.produceAccumulator.batchMaxBytes(this.batchMaxBytes); + } + + if (this.totalBatchMaxBytes > -1) { + this.produceAccumulator.totalBatchMaxBytes(this.totalBatchMaxBytes); + } + + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 4cf899f9708..33cf0df390d 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -68,6 +68,7 @@ import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -659,9 +660,9 @@ public void assertCreateDefaultMQProducer() { assertNotNull(producer1); assertEquals(producerGroupTemp, producer1.getProducerGroup()); assertNotNull(producer1.getDefaultMQProducerImpl()); - assertTrue(producer1.getTotalBatchMaxBytes() > 0); - assertTrue(producer1.getBatchMaxBytes() > 0); - assertTrue(producer1.getBatchMaxDelayMs() > 0); + assertEquals(0, producer1.getTotalBatchMaxBytes()); + assertEquals(0, producer1.getBatchMaxBytes()); + assertEquals(0, producer1.getBatchMaxDelayMs()); assertNull(producer1.getTopics()); assertFalse(producer1.isEnableTrace()); assertTrue(UtilAll.isBlank(producer1.getTraceTopic())); @@ -669,9 +670,9 @@ public void assertCreateDefaultMQProducer() { assertNotNull(producer2); assertEquals(producerGroupTemp, producer2.getProducerGroup()); assertNotNull(producer2.getDefaultMQProducerImpl()); - assertTrue(producer2.getTotalBatchMaxBytes() > 0); - assertTrue(producer2.getBatchMaxBytes() > 0); - assertTrue(producer2.getBatchMaxDelayMs() > 0); + assertEquals(0, producer2.getTotalBatchMaxBytes()); + assertEquals(0, producer2.getBatchMaxBytes()); + assertEquals(0, producer2.getBatchMaxDelayMs()); assertNull(producer2.getTopics()); assertFalse(producer2.isEnableTrace()); assertTrue(UtilAll.isBlank(producer2.getTraceTopic())); @@ -679,9 +680,9 @@ public void assertCreateDefaultMQProducer() { assertNotNull(producer3); assertEquals(producerGroupTemp, producer3.getProducerGroup()); assertNotNull(producer3.getDefaultMQProducerImpl()); - assertTrue(producer3.getTotalBatchMaxBytes() > 0); - assertTrue(producer3.getBatchMaxBytes() > 0); - assertTrue(producer3.getBatchMaxDelayMs() > 0); + assertEquals(0, producer3.getTotalBatchMaxBytes()); + assertEquals(0, producer3.getBatchMaxBytes()); + assertEquals(0, producer3.getBatchMaxDelayMs()); assertNotNull(producer3.getTopics()); assertEquals(1, producer3.getTopics().size()); assertFalse(producer3.isEnableTrace()); @@ -690,9 +691,9 @@ public void assertCreateDefaultMQProducer() { assertNotNull(producer4); assertEquals(producerGroupTemp, producer4.getProducerGroup()); assertNotNull(producer4.getDefaultMQProducerImpl()); - assertTrue(producer4.getTotalBatchMaxBytes() > 0); - assertTrue(producer4.getBatchMaxBytes() > 0); - assertTrue(producer4.getBatchMaxDelayMs() > 0); + assertEquals(0, producer4.getTotalBatchMaxBytes()); + assertEquals(0, producer4.getBatchMaxBytes()); + assertEquals(0, producer4.getBatchMaxDelayMs()); assertNull(producer4.getTopics()); assertTrue(producer4.isEnableTrace()); assertEquals("custom_trace_topic", producer4.getTraceTopic()); @@ -700,9 +701,9 @@ public void assertCreateDefaultMQProducer() { assertNotNull(producer5); assertEquals(producerGroupTemp, producer5.getProducerGroup()); assertNotNull(producer5.getDefaultMQProducerImpl()); - assertTrue(producer5.getTotalBatchMaxBytes() > 0); - assertTrue(producer5.getBatchMaxBytes() > 0); - assertTrue(producer5.getBatchMaxDelayMs() > 0); + assertEquals(0, producer5.getTotalBatchMaxBytes()); + assertEquals(0, producer5.getBatchMaxBytes()); + assertEquals(0, producer5.getBatchMaxDelayMs()); assertNotNull(producer5.getTopics()); assertEquals(1, producer5.getTopics().size()); assertTrue(producer5.isEnableTrace()); @@ -810,6 +811,136 @@ public void assertTotalBatchMaxBytes() throws NoSuchFieldException, IllegalAcces assertEquals(0L, producer.getTotalBatchMaxBytes()); } + @Test + public void assertProduceAccumulatorStart() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp); + assertEquals(0, producer.getTotalBatchMaxBytes()); + assertEquals(0, producer.getBatchMaxBytes()); + assertEquals(0, producer.getBatchMaxDelayMs()); + assertNull(getField(producer, "produceAccumulator", ProduceAccumulator.class)); + producer.start(); + assertTrue(producer.getTotalBatchMaxBytes() > 0); + assertTrue(producer.getBatchMaxBytes() > 0); + assertTrue(producer.getBatchMaxDelayMs() > 0); + assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class)); + } + + @Test + public void assertProduceAccumulatorBeforeStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp); + producer.totalBatchMaxBytes(64 * 1024 * 100); + producer.batchMaxBytes(64 * 1024); + producer.batchMaxDelayMs(10); + + producer.start(); + assertEquals(64 * 1024, producer.getBatchMaxBytes()); + assertEquals(10, producer.getBatchMaxDelayMs()); + assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class)); + } + + @Test + public void assertProduceAccumulatorAfterStartSet() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer = new DefaultMQProducer(producerGroupTemp); + producer.start(); + + assertNotNull(getField(producer, "produceAccumulator", ProduceAccumulator.class)); + + producer.totalBatchMaxBytes(64 * 1024 * 100); + producer.batchMaxBytes(64 * 1024); + producer.batchMaxDelayMs(10); + + assertEquals(64 * 1024, producer.getBatchMaxBytes()); + assertEquals(10, producer.getBatchMaxDelayMs()); + } + + @Test + public void assertProduceAccumulatorUnit() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp); + producer1.setUnitName("unit1"); + DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp); + producer2.setUnitName("unit2"); + + producer1.start(); + producer2.start(); + + ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class); + ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class); + + assertNotNull(producer1Accumulator); + assertNotNull(producer2Accumulator); + + assertNotEquals(producer1Accumulator, producer2Accumulator); + } + + @Test + public void assertProduceAccumulator() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp1 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1); + producer1.setInstanceName("instanceName1"); + String producerGroupTemp2 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2); + producer2.setInstanceName("instanceName2"); + + producer1.start(); + producer2.start(); + + ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class); + ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class); + + assertNotNull(producer1Accumulator); + assertNotNull(producer2Accumulator); + + assertNotEquals(producer1Accumulator, producer2Accumulator); + } + + @Test + public void assertProduceAccumulatorInstanceEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp1 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1); + producer1.setInstanceName("equalInstance"); + String producerGroupTemp2 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2); + producer2.setInstanceName("equalInstance"); + + producer1.start(); + producer2.start(); + + ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class); + ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class); + + assertNotNull(producer1Accumulator); + assertNotNull(producer2Accumulator); + + assertEquals(producer1Accumulator, producer2Accumulator); + } + + @Test + public void assertProduceAccumulatorInstanceAndUnitNameEqual() throws NoSuchFieldException, IllegalAccessException, MQClientException { + String producerGroupTemp1 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer1 = new DefaultMQProducer(producerGroupTemp1); + producer1.setInstanceName("equalInstance"); + producer1.setUnitName("equalUnitName"); + String producerGroupTemp2 = producerGroupPrefix + System.nanoTime(); + DefaultMQProducer producer2 = new DefaultMQProducer(producerGroupTemp2); + producer2.setInstanceName("equalInstance"); + producer2.setUnitName("equalUnitName"); + + producer1.start(); + producer2.start(); + + ProduceAccumulator producer1Accumulator = getField(producer1, "produceAccumulator", ProduceAccumulator.class); + ProduceAccumulator producer2Accumulator = getField(producer2, "produceAccumulator", ProduceAccumulator.class); + + assertNotNull(producer1Accumulator); + assertNotNull(producer2Accumulator); + + assertEquals(producer1Accumulator, producer2Accumulator); + } + @Test public void assertGetRetryResponseCodes() { assertNotNull(producer.getRetryResponseCodes()); @@ -875,4 +1006,11 @@ private void setField(final Object target, final String fieldName, final Object field.setAccessible(true); field.set(target, newValue); } + + private T getField(final Object target, final String fieldName, final Class fieldClassType) throws NoSuchFieldException, IllegalAccessException { + Class targetClazz = target.getClass(); + Field field = targetClazz.getDeclaredField(fieldName); + field.setAccessible(true); + return fieldClassType.cast(field.get(target)); + } } From 0c4064dc24d7d40485197713d61b341d6dbfb4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Fri, 11 Oct 2024 16:43:07 +0800 Subject: [PATCH 183/265] [ISSUE #8810] Fix independent execution of e2e and benchmark deployments (#8812) * Fix e2e and benchmark tests dependency issues * Update job naming * Fix bug * Update --- .github/workflows/push-ci.yml | 77 ++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index cc2a053eba3..9e13794b318 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -101,18 +101,16 @@ jobs: printf '%s\n' "${a[@]}" | jq -R . | jq -s . echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT - deploy: + deploy-e2e: if: ${{ success() }} - name: Deploy RocketMQ + name: Deploy RocketMQ For E2E needs: [list-version,docker] runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - test-type: [e2e, benchmark] steps: - - run: echo "Running ${{ matrix.test-type }}... " - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 name: Deploy rocketmq with: @@ -137,10 +135,44 @@ jobs: repository: ${{env.DOCKER_REPO}} tag: ${{ matrix.version }} + deploy-benchmark: + if: ${{ success() }} + name: Deploy RocketMQ For Benchmarking + needs: [list-version,docker] + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + version: ${{ fromJSON(needs.list-version.outputs.version-json) }} + steps: + - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 + name: Deploy rocketmq + with: + action: "deploy" + ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" + test-version: "${{ matrix.version }}" + chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git" + chart-branch: "master" + chart-path: "./rocketmq-k8s-helm" + job-id: "001-${{ strategy.job-index }}" + helm-values: | + nameserver: + image: + repository: ${{env.DOCKER_REPO}} + tag: ${{ matrix.version }} + broker: + image: + repository: ${{env.DOCKER_REPO}} + tag: ${{ matrix.version }} + proxy: + image: + repository: ${{env.DOCKER_REPO}} + tag: ${{ matrix.version }} + test-e2e-grpc-java: if: ${{ success() }} name: Test E2E grpc java - needs: [list-version, deploy] + needs: [list-version, deploy-e2e] runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -176,7 +208,7 @@ jobs: test-e2e-golang: if: ${{ success() }} name: Test E2E golang - needs: [list-version, deploy] + needs: [list-version, deploy-e2e] runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -217,7 +249,7 @@ jobs: test-e2e-remoting-java: if: ${{ success() }} name: Test E2E remoting java - needs: [ list-version, deploy ] + needs: [ list-version, deploy-e2e ] runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -254,7 +286,7 @@ jobs: if: ${{ success() }} runs-on: ubuntu-latest name: Performance benchmark test - needs: [ list-version, deploy ] + needs: [ list-version, deploy-benchmark ] timeout-minutes: 60 steps: - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e @@ -262,14 +294,14 @@ jobs: with: action: "performance-benchmark" ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" - job-id: 1 + job-id: "001-${{ strategy.job-index }}" # The time to run the test, 15 minutes test-time: "900" # Some thresholds set in advance min-send-tps-threshold: "12000" max-rt-ms-threshold: "500" avg-rt-ms-threshold: "10" - max-2c-rt-ms-threshold: "100" + max-2c-rt-ms-threshold: "150" avg-2c-rt-ms-threshold: "10" - name: Upload test report if: always() @@ -278,18 +310,16 @@ jobs: name: benchmark-report path: benchmark/ - clean: + clean-e2e: if: always() - name: Clean - needs: [list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java, benchmark-test] + name: Clean E2E + needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ] runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: version: ${{ fromJSON(needs.list-version.outputs.version-json) }} - test-type: [ e2e, benchmark ] steps: - - run: echo "Cleaning ${{ matrix.test-type }}... " - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 name: clean with: @@ -298,3 +328,20 @@ jobs: test-version: "${{ matrix.version }}" job-id: ${{ strategy.job-index }} + clean-benchmark: + if: always() + name: Clean Benchmarking + needs: [ list-version, benchmark-test ] + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + version: ${{ fromJSON(needs.list-version.outputs.version-json) }} + steps: + - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288 + name: clean + with: + action: "clean" + ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}" + test-version: "${{ matrix.version }}" + job-id: "001-${{ strategy.job-index }}" \ No newline at end of file From 2355a5f784bc54da1bffc00156e49a51db9dbd64 Mon Sep 17 00:00:00 2001 From: luozongle01 Date: Mon, 14 Oct 2024 09:13:17 +0800 Subject: [PATCH 184/265] [ISSUE #8816] Fix log typo. (#8817) --- .../org/apache/rocketmq/broker/controller/ReplicasManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java index d12d142d6d7..f22f22a12bd 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java @@ -525,7 +525,7 @@ private boolean applyBrokerId() { return true; } catch (Exception e) { - LOGGER.error("fail to apply broker id: {}", e, tempBrokerMetadata.getBrokerId()); + LOGGER.error("fail to apply broker id: {}", tempBrokerMetadata.getBrokerId(), e); return false; } } From 2fa7513a2bcdddd1583b7e293b2f06bd350691e0 Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:14:31 +0800 Subject: [PATCH 185/265] [ISSUE #8764] Implement consume lag estimation in cq rocksdb store (#8800) --- .../rocketmq/store/RocksDBMessageStore.java | 6 -- .../store/queue/RocksDBConsumeQueue.java | 41 ++++++++- .../store/queue/ConsumeQueueTest.java | 92 ++++++++++++++++++- 3 files changed, 128 insertions(+), 11 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 90df7aed596..21f8d45c9d9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -168,12 +168,6 @@ public void run() { } } - @Override - public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) { - // todo - return 0; - } - @Override public void initMetrics(Meter meter, Supplier attributesBuilderSupplier) { DefaultStoreMetricsManager.init(meter, attributesBuilderSupplier, this); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java index 2363c2896e5..83ba7bebad0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -222,10 +222,47 @@ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, Message @Override public long estimateMessageCount(long from, long to, MessageFilter filter) { - // todo - return 0; + // Check from and to offset validity + Pair fromUnit = getCqUnitAndStoreTime(from); + if (fromUnit == null) { + return -1; + } + + if (from >= to) { + return -1; + } + + if (to > getMaxOffsetInQueue()) { + to = getMaxOffsetInQueue(); + } + + int maxSampleSize = messageStore.getMessageStoreConfig().getMaxConsumeQueueScan(); + int sampleSize = to - from > maxSampleSize ? maxSampleSize : (int) (to - from); + + int matchThreshold = messageStore.getMessageStoreConfig().getSampleCountThreshold(); + int matchSize = 0; + + for (int i = 0; i < sampleSize; i++) { + long index = from + i; + Pair pair = getCqUnitAndStoreTime(index); + if (pair == null) { + continue; + } + CqUnit cqUnit = pair.getObject1(); + if (filter.isMatchedByConsumeQueue(cqUnit.getTagsCode(), cqUnit.getCqExtUnit())) { + matchSize++; + // if matchSize is plenty, early exit estimate + if (matchSize > matchThreshold) { + sampleSize = i; + break; + } + } + } + // Make sure the second half is a floating point number, otherwise it will be truncated to 0 + return sampleSize == 0 ? 0 : (long) ((to - from) * (matchSize / (sampleSize * 1.0))); } + @Override public long getMinOffsetInQueue() { return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId); diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java index c3c8be52ddd..bf3b1eeca83 100644 --- a/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/queue/ConsumeQueueTest.java @@ -22,6 +22,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.common.message.MessageDecoder; @@ -31,6 +32,7 @@ import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.junit.Assert; @@ -84,7 +86,26 @@ messageStoreConfig, new BrokerStatsManager(brokerConfig), return master; } - protected void putMsg(DefaultMessageStore messageStore) throws Exception { + protected RocksDBMessageStore genRocksdbMessageStore() throws Exception { + MessageStoreConfig messageStoreConfig = buildStoreConfig( + COMMIT_LOG_FILE_SIZE, CQ_FILE_SIZE, true, CQ_EXT_FILE_SIZE + ); + + BrokerConfig brokerConfig = new BrokerConfig(); + + RocksDBMessageStore master = new RocksDBMessageStore( + messageStoreConfig, new BrokerStatsManager(brokerConfig), + (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { + }, brokerConfig, new ConcurrentHashMap<>()); + + assertThat(master.load()).isTrue(); + + master.start(); + + return master; + } + + protected void putMsg(MessageStore messageStore) { int totalMsgs = 200; for (int i = 0; i < totalMsgs; i++) { MessageExtBrokerInner message = buildMessage(); @@ -184,9 +205,33 @@ public void testIterator() throws Exception { @Test public void testEstimateMessageCountInEmptyConsumeQueue() { - DefaultMessageStore master = null; + DefaultMessageStore messageStore = null; + try { + messageStore = gen(); + doTestEstimateMessageCountInEmptyConsumeQueue(messageStore); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + } + + @Test + public void testEstimateRocksdbMessageCountInEmptyConsumeQueue() { + if (notExecuted()) { + return; + } + DefaultMessageStore messageStore = null; + try { + messageStore = genRocksdbMessageStore(); + doTestEstimateMessageCountInEmptyConsumeQueue(messageStore); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + } + + public void doTestEstimateMessageCountInEmptyConsumeQueue(MessageStore master) { try { - master = gen(); ConsumeQueueInterface consumeQueue = master.findConsumeQueue(TOPIC, QUEUE_ID); MessageFilter filter = new MessageFilter() { @Override @@ -219,16 +264,34 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map pr } } + @Test + public void testEstimateRocksdbMessageCount() { + if (notExecuted()) { + return; + } + DefaultMessageStore messageStore = null; + try { + messageStore = genRocksdbMessageStore(); + doTestEstimateMessageCount(messageStore); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + } + @Test public void testEstimateMessageCount() { DefaultMessageStore messageStore = null; try { messageStore = gen(); + doTestEstimateMessageCount(messageStore); } catch (Exception e) { e.printStackTrace(); assertThat(Boolean.FALSE).isTrue(); } + } + public void doTestEstimateMessageCount(MessageStore messageStore) { try { try { putMsg(messageStore); @@ -265,15 +328,34 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map pr } } + @Test + public void testEstimateRocksdbMessageCountSample() { + if (notExecuted()) { + return; + } + DefaultMessageStore messageStore = null; + try { + messageStore = genRocksdbMessageStore(); + doTestEstimateMessageCountSample(messageStore); + } catch (Exception e) { + e.printStackTrace(); + assertThat(Boolean.FALSE).isTrue(); + } + } + @Test public void testEstimateMessageCountSample() { DefaultMessageStore messageStore = null; try { messageStore = gen(); + doTestEstimateMessageCountSample(messageStore); } catch (Exception e) { e.printStackTrace(); assertThat(Boolean.FALSE).isTrue(); } + } + + public void doTestEstimateMessageCountSample(MessageStore messageStore) { try { try { @@ -303,4 +385,8 @@ public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map pr UtilAll.deleteFile(new File(STORE_PATH)); } } + + private boolean notExecuted() { + return MixAll.isMac(); + } } From a7dc0235890641e23c65a9517c4800f989b85142 Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 16 Oct 2024 11:00:51 +0800 Subject: [PATCH 186/265] [ISSUE #8824] Fix IllegalStateException caused by logical errors (#8825) --- .../java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java index bca2d79d995..c8b404dd696 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java @@ -174,6 +174,7 @@ public void operationSucceed(RemotingCommand response) { PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); + break; default: RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); rpcResponsePromise.setSuccess(rpcResponse); From 4f5f705f16faeb7d491b25679d1c100f38264bb9 Mon Sep 17 00:00:00 2001 From: rongtong Date: Wed, 16 Oct 2024 13:39:27 +0800 Subject: [PATCH 187/265] [ISSUE #8780] Implement asynchronous storage of ack/ck messages in pop consume to enhance performance (#8727) * Pop consume asynchronization * Pass UTs and ITs * Pass the checkstyle * Fix LocalGrpcIT can not pass * Fix the UT can not pass * Simplify duplicate methods in EscapeBridge --- .../broker/failover/EscapeBridge.java | 62 +++++--- .../broker/processor/AckMessageProcessor.java | 39 +++-- .../ChangeInvisibleTimeProcessor.java | 149 +++++++++++------- .../processor/PopBufferMergeService.java | 113 +++++++++---- .../ChangeInvisibleTimeProcessorTest.java | 3 +- .../apache/rocketmq/common/BrokerConfig.java | 20 +++ .../service/message/LocalMessageService.java | 8 +- .../message/LocalMessageServiceTest.java | 5 +- .../test/base/IntegrationTestBase.java | 2 + 9 files changed, 275 insertions(+), 126 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java index 762d917d640..dd37f42b2c5 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java @@ -137,7 +137,7 @@ public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, Str brokerNameToSend = mqSelected.getBrokerName(); if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { LOG.warn("putMessageToRemoteBroker failed, remote broker not found. Topic: {}, MsgId: {}, Broker: {}", - messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); + messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); return null; } } else { @@ -147,7 +147,7 @@ public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, Str final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend); if (null == brokerAddrToSend) { LOG.warn("putMessageToRemoteBroker failed, remote broker address not found. Topic: {}, MsgId: {}, Broker: {}", - messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); + messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend); return null; } @@ -197,7 +197,7 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner producerGroup, SEND_TIMEOUT); return future.exceptionally(throwable -> null) - .thenApplyAsync(sendResult -> transformSendResult2PutResult(sendResult), this.defaultAsyncSenderExecutor) + .thenApplyAsync(this::transformSendResult2PutResult, this.defaultAsyncSenderExecutor) .exceptionally(throwable -> transformSendResult2PutResult(null)); } catch (Exception e) { @@ -211,7 +211,6 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner } } - private String getProducerGroup(MessageExtBrokerInner messageExt) { if (null == messageExt) { return this.innerProducerGroupName; @@ -223,12 +222,29 @@ private String getProducerGroup(MessageExtBrokerInner messageExt) { return producerGroup; } - public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageExt) { BrokerController masterBroker = this.brokerController.peekMasterBroker(); if (masterBroker != null) { return masterBroker.getMessageStore().putMessage(messageExt); - } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster() + } + try { + return asyncRemotePutMessageToSpecificQueue(messageExt).get(SEND_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (Exception e) { + LOG.error("Put message to specific queue error", e); + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, true); + } + } + + public CompletableFuture asyncPutMessageToSpecificQueue(MessageExtBrokerInner messageExt) { + BrokerController masterBroker = this.brokerController.peekMasterBroker(); + if (masterBroker != null) { + return masterBroker.getMessageStore().asyncPutMessage(messageExt); + } + return asyncRemotePutMessageToSpecificQueue(messageExt); + } + + public CompletableFuture asyncRemotePutMessageToSpecificQueue(MessageExtBrokerInner messageExt) { + if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster() && this.brokerController.getBrokerConfig().isEnableRemoteEscape()) { try { messageExt.setWaitStoreMsgOK(false); @@ -237,7 +253,7 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE List mqs = topicPublishInfo.getMessageQueueList(); if (null == mqs || mqs.isEmpty()) { - return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true)); } String id = messageExt.getTopic() + messageExt.getStoreHost(); @@ -248,19 +264,17 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE String brokerNameToSend = mq.getBrokerName(); String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend); - final SendResult sendResult = this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker( + return this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBrokerAsync( brokerAddrToSend, brokerNameToSend, - messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT); - - return transformSendResult2PutResult(sendResult); + messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT).thenCompose(sendResult -> CompletableFuture.completedFuture(transformSendResult2PutResult(sendResult))); } catch (Exception e) { LOG.error("sendMessageInFailover to remote failed", e); - return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true)); } } else { LOG.warn("Put message to specific queue failed, enableSlaveActingMaster={}, enableRemoteEscape={}.", this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape()); - return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null)); } } @@ -282,12 +296,14 @@ private PutMessageResult transformSendResult2PutResult(SendResult sendResult) { } } - public Triple getMessage(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { + public Triple getMessage(String topic, long offset, int queueId, String brokerName, + boolean deCompressBody) { return getMessageAsync(topic, offset, queueId, brokerName, deCompressBody).join(); } // Triple, check info and retry if and only if MessageExt is null - public CompletableFuture> getMessageAsync(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) { + public CompletableFuture> getMessageAsync(String topic, long offset, + int queueId, String brokerName, boolean deCompressBody) { MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName); if (messageStore != null) { return messageStore.getMessageAsync(innerConsumerGroupName, topic, queueId, offset, 1, null) @@ -300,9 +316,9 @@ public CompletableFuture> getMessageAsync(St if (list == null || list.isEmpty()) { // OFFSET_FOUND_NULL returned by TieredMessageStore indicates exception occurred boolean needRetry = GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus()) - && messageStore instanceof TieredMessageStore; + && messageStore instanceof TieredMessageStore; LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, needRetry {}, result is {}", - topic, offset, queueId, needRetry, result); + topic, offset, queueId, needRetry, result); return Triple.of(null, "Can not get msg", needRetry); } return Triple.of(list.get(0), "", false); @@ -340,12 +356,14 @@ protected List decodeMsgList(GetMessageResult getMessageResult, bool return foundList; } - protected Triple getMessageFromRemote(String topic, long offset, int queueId, String brokerName) { + protected Triple getMessageFromRemote(String topic, long offset, int queueId, + String brokerName) { return getMessageFromRemoteAsync(topic, offset, queueId, brokerName).join(); } // Triple, check info and retry if and only if MessageExt is null - protected CompletableFuture> getMessageFromRemoteAsync(String topic, long offset, int queueId, String brokerName) { + protected CompletableFuture> getMessageFromRemoteAsync(String topic, + long offset, int queueId, String brokerName) { try { String brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false); if (null == brokerAddr) { @@ -359,11 +377,11 @@ protected CompletableFuture> getMessageFromR } return this.brokerController.getBrokerOuterAPI().pullMessageFromSpecificBrokerAsync(brokerName, - brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS) + brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS) .thenApply(pullResult -> { if (pullResult.getLeft() != null - && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus()) - && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) { + && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus()) + && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) { return Triple.of(pullResult.getLeft().getMsgFoundList().get(0), "", false); } return Triple.of(null, pullResult.getMiddle(), pullResult.getRight()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index 6f7b7e8a24e..dc1b1b53a32 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -98,7 +98,7 @@ public boolean isPopReviveServiceRunning() { @Override public RemotingCommand processRequest(final ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + RemotingCommand request) throws RemotingCommandException { return this.processRequest(ctx.channel(), request, true); } @@ -108,7 +108,7 @@ public boolean rejectRequest() { } private RemotingCommand processRequest(final Channel channel, RemotingCommand request, - boolean brokerAllowSuspend) throws RemotingCommandException { + boolean brokerAllowSuspend) throws RemotingCommandException { AckMessageRequestHeader requestHeader; BatchAckMessageRequestBody reqBody = null; final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); @@ -126,7 +126,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) { String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", - requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); + requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); POP_LOGGER.warn(errorInfo); response.setCode(ResponseCode.MESSAGE_ILLEGAL); response.setRemark(errorInfo); @@ -137,7 +137,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d", - requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset); + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset); POP_LOGGER.warn(errorInfo); response.setCode(ResponseCode.NO_MESSAGE); response.setRemark(errorInfo); @@ -165,7 +165,8 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re return response; } - private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, final RemotingCommand response, final Channel channel, String brokerName) { + private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, + final RemotingCommand response, final Channel channel, String brokerName) { String[] extraInfo; String consumeGroup, topic; int qId, rqId; @@ -268,18 +269,36 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA msgInner.setDeliverTimeMs(popTime + invisibleTime); msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + if (brokerController.getBrokerConfig().isAppendAckAsync()) { + int finalAckCount = ackCount; + this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> { + handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, finalAckCount); + }).exceptionally(throwable -> { + handlePutMessageResult(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, false), + ackMsg, topic, consumeGroup, popTime, qId, finalAckCount); + POP_LOGGER.error("put ack msg error ", throwable); + return null; + }); + } else { + PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, ackCount); + } + } + + private void handlePutMessageResult(PutMessageResult putMessageResult, AckMsg ackMsg, String topic, + String consumeGroup, long popTime, int qId, int ackCount) { if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { POP_LOGGER.error("put ack msg error:" + putMessageResult); } PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); } - protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) { + protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, + long invisibleTime, Channel channel, RemotingCommand response) { String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); if (ackOffset < oldOffset) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index bdfffff096a..af3b8ae6f05 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -19,6 +19,8 @@ import com.alibaba.fastjson.JSON; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.PopAckConstants; @@ -33,13 +35,13 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; -import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -67,6 +69,35 @@ public boolean rejectRequest() { private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend) throws RemotingCommandException { + + CompletableFuture responseFuture = processRequestAsync(channel, request, brokerAllowSuspend); + + if (brokerController.getBrokerConfig().isAppendCkAsync() && brokerController.getBrokerConfig().isAppendAckAsync()) { + responseFuture.thenAccept(response -> doResponse(channel, request, response)).exceptionally(throwable -> { + RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setOpaque(request.getOpaque()); + doResponse(channel, request, response); + POP_LOGGER.error("append checkpoint or ack origin failed", throwable); + return null; + }); + } else { + RemotingCommand response; + try { + response = responseFuture.get(3000, TimeUnit.MILLISECONDS); + } catch (Exception e) { + response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setOpaque(request.getOpaque()); + POP_LOGGER.error("append checkpoint or ack origin failed", e); + } + return response; + } + return null; + } + + public CompletableFuture processRequestAsync(final Channel channel, RemotingCommand request, + boolean brokerAllowSuspend) throws RemotingCommandException { final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class); RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); response.setCode(ResponseCode.SUCCESS); @@ -77,7 +108,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel)); response.setCode(ResponseCode.TOPIC_NOT_EXIST); response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); - return response; + return CompletableFuture.completedFuture(response); } if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) { @@ -86,46 +117,35 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re POP_LOGGER.warn(errorInfo); response.setCode(ResponseCode.MESSAGE_ILLEGAL); response.setRemark(errorInfo); - return response; + return CompletableFuture.completedFuture(response); } long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { response.setCode(ResponseCode.NO_MESSAGE); - return response; + return CompletableFuture.completedFuture(response); } String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo()); if (ExtraInfoUtil.isOrder(extraInfo)) { - return processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader); + return CompletableFuture.completedFuture(processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader)); } // add new ck long now = System.currentTimeMillis(); - PutMessageResult ckResult = appendCheckPoint(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, ExtraInfoUtil.getBrokerName(extraInfo)); - - if (ckResult.getPutMessageStatus() != PutMessageStatus.PUT_OK - && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT - && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT - && ckResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { - POP_LOGGER.error("change Invisible, put new ck error: {}", ckResult); - response.setCode(ResponseCode.SYSTEM_ERROR); - return response; - } - - // ack old msg. - try { - ackOrigin(requestHeader, extraInfo); - } catch (Throwable e) { - POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage()); - // cancel new ck? - } - responseHeader.setInvisibleTime(requestHeader.getInvisibleTime()); - responseHeader.setPopTime(now); - responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo)); - return response; + CompletableFuture futureResult = appendCheckPointThenAckOrigin(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, extraInfo); + return futureResult.thenCompose(result -> { + if (result) { + responseHeader.setInvisibleTime(requestHeader.getInvisibleTime()); + responseHeader.setPopTime(now); + responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo)); + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + } + return CompletableFuture.completedFuture(response); + }); } protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTimeRequestHeader requestHeader, @@ -158,7 +178,8 @@ protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTime return response; } - private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo) { + private CompletableFuture ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, + String[] extraInfo) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); AckMsg ackMsg = new AckMsg(); @@ -176,7 +197,7 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1); if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) { - return; + return CompletableFuture.completedFuture(true); } msgInner.setTopic(reviveTopic); @@ -189,18 +210,25 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str msgInner.setDeliverTimeMs(ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo)); msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); - if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { - POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult); - } - PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); + return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> { + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult); + } + PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); + return CompletableFuture.completedFuture(true); + }).exceptionally(e -> { + POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage()); + return false; + }); } - private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader requestHeader, int reviveQid, - int queueId, long offset, long popTime, String brokerName) { + private CompletableFuture appendCheckPointThenAckOrigin( + final ChangeInvisibleTimeRequestHeader requestHeader, + int reviveQid, + int queueId, long offset, long popTime, String[] extraInfo) { // add check point msg to revive log MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); @@ -214,7 +242,7 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader ck.setTopic(requestHeader.getTopic()); ck.setQueueId(queueId); ck.addDiff(0); - ck.setBrokerName(brokerName); + ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo)); msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(reviveQid); @@ -225,21 +253,36 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval); msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck)); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); - - if (brokerController.getBrokerConfig().isEnablePopLog()) { - POP_LOGGER.info("change Invisible , appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset, - ck.getReviveTime(), putMessageResult); - } + return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> { + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("change Invisible, appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset, + ck.getReviveTime(), putMessageResult); + } - if (putMessageResult != null) { - PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus()); - if (putMessageResult.isOk()) { - this.brokerController.getBrokerStatsManager().incBrokerCkNums(1); - this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1); + if (putMessageResult != null) { + PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus()); + if (putMessageResult.isOk()) { + this.brokerController.getBrokerStatsManager().incBrokerCkNums(1); + this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1); + } } - } + if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + POP_LOGGER.error("change invisible, put new ck error: {}", putMessageResult); + return CompletableFuture.completedFuture(false); + } else { + return ackOrigin(requestHeader, extraInfo); + } + }).exceptionally(throwable -> { + POP_LOGGER.error("change invisible, put new ck error", throwable); + return null; + }); + } - return putMessageResult; + protected void doResponse(Channel channel, RemotingCommand request, + final RemotingCommand response) { + NettyRemotingAbstract.writeResponse(channel, request, response); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java index 8a85dd8fec8..e05ab8ebeae 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java @@ -216,7 +216,8 @@ private void scanGarbage() { private void scan() { long startTime = System.currentTimeMillis(); - int count = 0, countCk = 0; + AtomicInteger count = new AtomicInteger(0); + int countCk = 0; Iterator> iterator = buffer.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -257,14 +258,14 @@ private void scan() { } else if (pointWrapper.isJustOffset()) { // just offset should be in store. if (pointWrapper.getReviveQueueOffset() < 0) { - putCkToStore(pointWrapper, false); + putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync()); countCk++; } continue; } else if (removeCk) { // put buffer ak to store if (pointWrapper.getReviveQueueOffset() < 0) { - putCkToStore(pointWrapper, false); + putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync()); countCk++; } @@ -278,17 +279,12 @@ private void scan() { for (byte i = 0; i < point.getNum(); i++) { // reput buffer ak to store if (DataConverter.getBit(pointWrapper.getBits().get(), i) - && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) { + && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) { indexList.add(i); } } if (indexList.size() > 0) { - if (putBatchAckToStore(pointWrapper, indexList)) { - count += indexList.size(); - for (Byte i : indexList) { - markBitCAS(pointWrapper.getToStoreBits(), i); - } - } + putBatchAckToStore(pointWrapper, indexList, count); } } finally { indexList.clear(); @@ -297,11 +293,8 @@ private void scan() { for (byte i = 0; i < point.getNum(); i++) { // reput buffer ak to store if (DataConverter.getBit(pointWrapper.getBits().get(), i) - && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) { - if (putAckToStore(pointWrapper, i)) { - count++; - markBitCAS(pointWrapper.getToStoreBits(), i); - } + && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) { + putAckToStore(pointWrapper, i, count); } } } @@ -312,7 +305,6 @@ private void scan() { } iterator.remove(); counter.decrementAndGet(); - continue; } } } @@ -323,13 +315,13 @@ private void scan() { if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) { POP_LOGGER.warn("[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, " + "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}", - eclipse, count, countCk, counter.get(), offsetBufferSize); + eclipse, count.get(), countCk, counter.get(), offsetBufferSize); this.serving = false; } else { if (scanTimes % countOfSecond1 == 0) { POP_LOGGER.info("[PopBuffer]scan, PopBufferEclipse={}, " + "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}", - eclipse, count, countCk, counter.get(), offsetBufferSize); + eclipse, count.get(), countCk, counter.get(), offsetBufferSize); } } PopMetricsManager.recordPopBufferScanTimeConsume(eclipse); @@ -429,7 +421,8 @@ private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) { * @param nextBeginOffset * @return */ - public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) { + public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, + long nextBeginOffset) { PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true); if (this.buffer.containsKey(pointWrapper.getMergeKey())) { @@ -439,7 +432,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi return false; } - this.putCkToStore(pointWrapper, !checkQueueOk(pointWrapper)); + this.putCkToStore(pointWrapper, checkQueueOk(pointWrapper)); putOffsetQueue(pointWrapper); this.buffer.put(pointWrapper.getMergeKey(), pointWrapper); @@ -447,7 +440,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("[PopBuffer]add ck just offset, {}", pointWrapper); } - return true; + return true; } public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime, @@ -597,13 +590,32 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean if (pointWrapper.getReviveQueueOffset() >= 0) { return; } + MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId()); - PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + + // Indicates that ck message is storing + pointWrapper.setReviveQueueOffset(Long.MAX_VALUE); + if (brokerController.getBrokerConfig().isAppendCkAsync() && runInCurrent) { + brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> { + handleCkMessagePutResult(putMessageResult, pointWrapper); + }).exceptionally(throwable -> { + POP_LOGGER.error("[PopBuffer]put ck to store fail: {}", pointWrapper, throwable); + pointWrapper.setReviveQueueOffset(-1); + return null; + }); + } else { + PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + handleCkMessagePutResult(putMessageResult, pointWrapper); + } + } + + private void handleCkMessagePutResult(PutMessageResult putMessageResult, final PopCheckPointWrapper pointWrapper) { PopMetricsManager.incPopReviveCkPutCount(pointWrapper.getCk(), putMessageResult.getPutMessageStatus()); if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + pointWrapper.setReviveQueueOffset(-1); POP_LOGGER.error("[PopBuffer]put ck to store fail: {}, {}", pointWrapper, putMessageResult); return; } @@ -621,7 +633,7 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean } } - private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex) { + private void putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex, AtomicInteger count) { PopCheckPoint point = pointWrapper.getCk(); MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); final AckMsg ackMsg = new AckMsg(); @@ -643,23 +655,39 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg)); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + + if (brokerController.getBrokerConfig().isAppendAckAsync()) { + brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> { + handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex); + }).exceptionally(throwable -> { + POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}", pointWrapper, ackMsg, throwable); + return null; + }); + } else { + PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex); + } + } + + private void handleAckPutMessageResult(AckMsg ackMsg, PutMessageResult putMessageResult, + PopCheckPointWrapper pointWrapper, AtomicInteger count, byte msgIndex) { PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}, {}", pointWrapper, ackMsg, putMessageResult); - return false; + return; } if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("[PopBuffer]put ack to store ok: {}, {}, {}", pointWrapper, ackMsg, putMessageResult); } - - return true; + count.incrementAndGet(); + markBitCAS(pointWrapper.getToStoreBits(), msgIndex); } - private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List msgIndexList) { + private void putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List msgIndexList, + AtomicInteger count) { PopCheckPoint point = pointWrapper.getCk(); MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); final BatchAckMsg batchAckMsg = new BatchAckMsg(); @@ -683,19 +711,36 @@ private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, fina msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genBatchAckUniqueId(batchAckMsg)); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + if (brokerController.getBrokerConfig().isAppendAckAsync()) { + brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> { + handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList); + }).exceptionally(throwable -> { + POP_LOGGER.error("[PopBuffer]put batchAckMsg to store fail: {}, {}", pointWrapper, batchAckMsg, throwable); + return null; + }); + } else { + PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList); + } + } + + private void handleBatchAckPutMessageResult(BatchAckMsg batchAckMsg, PutMessageResult putMessageResult, + PopCheckPointWrapper pointWrapper, AtomicInteger count, List msgIndexList) { if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT - && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT + && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) { POP_LOGGER.error("[PopBuffer]put batch ack to store fail: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult); - return false; + return; } if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("[PopBuffer]put batch ack to store ok: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult); } - return true; + count.addAndGet(msgIndexList.size()); + for (Byte i : msgIndexList) { + markBitCAS(pointWrapper.getToStoreBits(), i); + } } private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java index ee11f046d01..a7aae7ee3dc 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java @@ -19,6 +19,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.net.Broker2Client; @@ -108,7 +109,7 @@ public void init() throws IllegalAccessException, NoSuchFieldException { @Test public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException { - when(escapeBridge.putMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + when(escapeBridge.asyncPutMessageToSpecificQueue(any(MessageExtBrokerInner.class))).thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)))); int queueId = 0; long queueOffset = 0; long popTime = System.currentTimeMillis() - 1_000; diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 2123e9b339d..2acfdd69a5c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -427,6 +427,10 @@ public class BrokerConfig extends BrokerIdentity { // if false, will still rewrite ck after max times 17 private boolean skipWhenCKRePutReachMaxTimes = false; + private boolean appendAckAsync = false; + + private boolean appendCkAsync = false; + public String getConfigBlackList() { return configBlackList; } @@ -1859,4 +1863,20 @@ public int getUpdateNameServerAddrPeriod() { public void setUpdateNameServerAddrPeriod(int updateNameServerAddrPeriod) { this.updateNameServerAddrPeriod = updateNameServerAddrPeriod; } + + public boolean isAppendAckAsync() { + return appendAckAsync; + } + + public void setAppendAckAsync(boolean appendAckAsync) { + this.appendAckAsync = appendAckAsync; + } + + public boolean isAppendCkAsync() { + return appendCkAsync; + } + + public void setAppendCkAsync(boolean appendCkAsync) { + this.appendCkAsync = appendCkAsync; + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index 6b2ba02f7c9..a8088a95d0a 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -176,7 +176,8 @@ public CompletableFuture sendMessageBack(ProxyContext ctx, Rece } @Override - public CompletableFuture endTransactionOneway(ProxyContext ctx, String brokerName, EndTransactionRequestHeader requestHeader, + public CompletableFuture endTransactionOneway(ProxyContext ctx, String brokerName, + EndTransactionRequestHeader requestHeader, long timeoutMillis) { CompletableFuture future = new CompletableFuture<>(); SimpleChannel channel = channelManager.createChannel(ctx); @@ -310,9 +311,8 @@ public CompletableFuture changeInvisibleTime(ProxyContext ctx, Receip RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader, ctx.getLanguage()); CompletableFuture future = new CompletableFuture<>(); try { - RemotingCommand response = brokerController.getChangeInvisibleTimeProcessor() - .processRequest(channelHandlerContext, command); - future.complete(response); + future = brokerController.getChangeInvisibleTimeProcessor() + .processRequestAsync(channelHandlerContext.channel(), command, true); } catch (Exception e) { log.error("Fail to process changeInvisibleTime command", e); future.completeExceptionally(e); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java index 3e3d37086b5..f7a656d7682 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.proxy.service.message; +import io.netty.channel.Channel; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -370,11 +371,11 @@ public void testChangeInvisibleTime() throws Exception { responseHeader.setReviveQid(newReviveQueueId); responseHeader.setInvisibleTime(newInvisibleTime); responseHeader.setPopTime(newPopTime); - Mockito.when(changeInvisibleTimeProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + Mockito.when(changeInvisibleTimeProcessorMock.processRequestAsync(Mockito.any(Channel.class), Mockito.argThat(argument -> { boolean first = argument.getCode() == RequestCode.CHANGE_MESSAGE_INVISIBLETIME; boolean second = argument.readCustomHeader() instanceof ChangeInvisibleTimeRequestHeader; return first && second; - }))).thenReturn(remotingCommand); + }), Mockito.any(Boolean.class))).thenReturn(CompletableFuture.completedFuture(remotingCommand)); ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader(); CompletableFuture future = localMessageService.changeInvisibleTime(proxyContext, handle, messageId, requestHeader, 1000L); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 2217936929c..fde991ad13d 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -136,6 +136,8 @@ public static BrokerController createAndStartBroker(String nsAddr) { brokerConfig.setNamesrvAddr(nsAddr); brokerConfig.setEnablePropertyFilter(true); brokerConfig.setEnableCalcFilterBitMap(true); + brokerConfig.setAppendAckAsync(true); + brokerConfig.setAppendCkAsync(true); storeConfig.setEnableConsumeQueueExt(true); brokerConfig.setLoadBalancePollNameServerInterval(500); storeConfig.setStorePathRootDir(baseDir); From 031687dbd78df0c52f1b2a73a7e12b203453128f Mon Sep 17 00:00:00 2001 From: Ji Juntao Date: Thu, 17 Oct 2024 19:16:14 +0800 Subject: [PATCH 188/265] [ISSUE #8835] When ck is in the buffer, incomplete ack will lead to message duplication. (#8836) * add brokerName in ackMsg * add brokerName in ackMsg --- .../apache/rocketmq/broker/processor/PopBufferMergeService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java index e05ab8ebeae..9f10b483ddb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java @@ -644,6 +644,7 @@ private void putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgInde ackMsg.setTopic(point.getTopic()); ackMsg.setQueueId(point.getQueueId()); ackMsg.setPopTime(point.getPopTime()); + ackMsg.setBrokerName(point.getBrokerName()); msgInner.setTopic(popMessageProcessor.reviveTopic); msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); msgInner.setQueueId(pointWrapper.getReviveQueueId()); From 01f9848b74bc11b28d25ca0be6a15875d1c5df7c Mon Sep 17 00:00:00 2001 From: mawen12 <1181963012mw@gmail.com> Date: Fri, 18 Oct 2024 09:18:41 +0800 Subject: [PATCH 189/265] fix variables match annotation (@RocketMQResource) (#8821) --- .../remoting/protocol/header/ResetOffsetRequestHeader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java index de9432ca515..f72fe57136c 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ResetOffsetRequestHeader.java @@ -31,11 +31,11 @@ public class ResetOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull @RocketMQResource(ResourceType.GROUP) - private String topic; + private String group; @CFNotNull @RocketMQResource(ResourceType.TOPIC) - private String group; + private String topic; private int queueId = -1; From 49e23e1e0b8bc37e4497de97202a343956de3968 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Fri, 18 Oct 2024 14:30:38 +0800 Subject: [PATCH 190/265] fix: atomic flush incorrect use and clean up code (#8830) Signed-off-by: Li Zhanhui --- .../common/config/AbstractRocksDBStorage.java | 199 +++++++++++++----- .../rocketmq/common/config/ConfigHelper.java | 121 +++++++++++ .../common/config/ConfigRocksDBStorage.java | 166 +-------------- .../store/queue/RocksDBConsumeQueueStore.java | 8 +- .../rocksdb/ConsumeQueueRocksDBStorage.java | 41 ++-- .../store/rocksdb/RocksDBOptionsFactory.java | 10 +- 6 files changed, 299 insertions(+), 246 deletions(-) create mode 100644 common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index 13522889bb3..42ddbdc728c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -17,18 +17,10 @@ package org.apache.rocketmq.common.config; import com.google.common.collect.Maps; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import io.netty.buffer.PooledByteBufAllocator; +import java.nio.charset.StandardCharsets; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -37,7 +29,9 @@ import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.CompactRangeOptions; import org.rocksdb.CompactionOptions; +import org.rocksdb.CompressionType; import org.rocksdb.DBOptions; +import org.rocksdb.Env; import org.rocksdb.FlushOptions; import org.rocksdb.LiveFileMetaData; import org.rocksdb.Priority; @@ -49,14 +43,31 @@ import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; -import static org.rocksdb.RocksDB.NOT_FOUND; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public abstract class AbstractRocksDBStorage { protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + /** + * Direct Jemalloc allocator + */ + public static final PooledByteBufAllocator POOLED_ALLOCATOR = new PooledByteBufAllocator(true); + + public static final byte CTRL_0 = '\u0000'; + public static final byte CTRL_1 = '\u0001'; + public static final byte CTRL_2 = '\u0002'; + private static final String SPACE = " | "; - protected String dbPath; + protected final String dbPath; protected boolean readOnly; protected RocksDB db; protected DBOptions options; @@ -71,7 +82,8 @@ public abstract class AbstractRocksDBStorage { protected CompactRangeOptions compactRangeOptions; protected ColumnFamilyHandle defaultCFHandle; - protected final List cfOptions = new ArrayList(); + protected final List cfOptions = new ArrayList<>(); + protected final List cfHandles = new ArrayList<>(); protected volatile boolean loaded; private volatile boolean closed; @@ -79,15 +91,76 @@ public abstract class AbstractRocksDBStorage { private final Semaphore reloadPermit = new Semaphore(1); private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( - 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, - new ArrayBlockingQueue(1), - new ThreadFactoryImpl("RocksDBManualCompactionService_"), - new ThreadPoolExecutor.DiscardOldestPolicy()); + 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1), + new ThreadFactoryImpl("RocksDBManualCompactionService_"), + new ThreadPoolExecutor.DiscardOldestPolicy()); static { RocksDB.loadLibrary(); } + public AbstractRocksDBStorage(String dbPath) { + this.dbPath = dbPath; + } + + protected void initOptions() { + initWriteOptions(); + initAbleWalWriteOptions(); + initReadOptions(); + initTotalOrderReadOptions(); + initCompactRangeOptions(); + initCompactionOptions(); + } + + /** + * Write options for Atomic Flush + */ + protected void initWriteOptions() { + this.writeOptions = new WriteOptions(); + this.writeOptions.setSync(false); + this.writeOptions.setDisableWAL(true); + this.writeOptions.setNoSlowdown(true); + } + + protected void initAbleWalWriteOptions() { + this.ableWalWriteOptions = new WriteOptions(); + this.ableWalWriteOptions.setSync(false); + this.ableWalWriteOptions.setDisableWAL(false); + this.ableWalWriteOptions.setNoSlowdown(true); + } + + protected void initReadOptions() { + this.readOptions = new ReadOptions(); + this.readOptions.setPrefixSameAsStart(true); + this.readOptions.setTotalOrderSeek(false); + this.readOptions.setTailing(false); + } + + protected void initTotalOrderReadOptions() { + this.totalOrderReadOptions = new ReadOptions(); + this.totalOrderReadOptions.setPrefixSameAsStart(false); + this.totalOrderReadOptions.setTotalOrderSeek(true); + this.totalOrderReadOptions.setTailing(false); + } + + protected void initCompactRangeOptions() { + this.compactRangeOptions = new CompactRangeOptions(); + this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); + this.compactRangeOptions.setAllowWriteStall(true); + this.compactRangeOptions.setExclusiveManualCompaction(false); + this.compactRangeOptions.setChangeLevel(true); + this.compactRangeOptions.setTargetLevel(-1); + this.compactRangeOptions.setMaxSubcompactions(4); + } + + protected void initCompactionOptions() { + this.compactionOptions = new CompactionOptions(); + this.compactionOptions.setCompression(CompressionType.LZ4_COMPRESSION); + this.compactionOptions.setMaxSubcompactions(4); + this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L); + } + public boolean hold() { if (!this.loaded || this.db == null || this.closed) { LOGGER.error("hold rocksdb Failed. {}", this.dbPath); @@ -101,8 +174,8 @@ public void release() { } protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, - final byte[] keyBytes, final int keyLen, - final byte[] valueBytes, final int valueLen) throws RocksDBException { + final byte[] keyBytes, final int keyLen, + final byte[] valueBytes, final int valueLen) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -118,7 +191,7 @@ protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, } protected void put(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, - final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException { + final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -159,13 +232,13 @@ protected byte[] get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, byte[ } } - protected boolean get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, - final ByteBuffer keyBB, final ByteBuffer valueBB) throws RocksDBException { + protected int get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, final ByteBuffer keyBB, + final ByteBuffer valueBB) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } try { - return this.db.get(cfHandle, readOptions, keyBB, valueBB) != NOT_FOUND; + return this.db.get(cfHandle, readOptions, keyBB, valueBB); } catch (RocksDBException e) { LOGGER.error("get Failed. {}, {}", this.dbPath, getStatusError(e)); throw e; @@ -175,8 +248,8 @@ protected boolean get(ColumnFamilyHandle cfHandle, ReadOptions readOptions, } protected List multiGet(final ReadOptions readOptions, - final List columnFamilyHandleList, - final List keys) throws RocksDBException { + final List columnFamilyHandleList, + final List keys) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -190,7 +263,8 @@ protected List multiGet(final ReadOptions readOptions, } } - protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, byte[] keyBytes) throws RocksDBException { + protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, + byte[] keyBytes) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -204,7 +278,8 @@ protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, by } } - protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, ByteBuffer keyBB) throws RocksDBException { + protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, ByteBuffer keyBB) + throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -218,8 +293,8 @@ protected void delete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, By } } - protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, - final byte[] startKey, final byte[] endKey) throws RocksDBException { + protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, final byte[] startKey, + final byte[] endKey) throws RocksDBException { if (!hold()) { throw new IllegalStateException("rocksDB:" + this + " is not ready"); } @@ -262,16 +337,17 @@ public void run() { }); } - protected void open(final List cfDescriptors, - final List cfHandles) throws RocksDBException { + protected void open(final List cfDescriptors) throws RocksDBException { + this.cfHandles.clear(); if (this.readOnly) { this.db = RocksDB.openReadOnly(this.options, this.dbPath, cfDescriptors, cfHandles); } else { this.db = RocksDB.open(this.options, this.dbPath, cfDescriptors, cfHandles); } - this.db.getEnv().setBackgroundThreads(8, Priority.HIGH); - this.db.getEnv().setBackgroundThreads(8, Priority.LOW); - + assert cfDescriptors.size() == cfHandles.size(); + try (Env env = this.db.getEnv()) { + env.setBackgroundThreads(8, Priority.LOW); + } if (this.db == null) { throw new RocksDBException("open rocksdb null"); } @@ -293,6 +369,9 @@ public synchronized boolean start() { } } + /** + * Close column family handles except the default column family + */ protected abstract void preShutdown(); public synchronized boolean shutdown() { @@ -310,11 +389,12 @@ public synchronized boolean shutdown() { } this.db.cancelAllBackgroundWork(true); this.db.pauseBackgroundWork(); - //The close order is matter. + //The close order matters. //1. close column family handles preShutdown(); this.defaultCFHandle.close(); + //2. close column family options. for (final ColumnFamilyOptions opt : this.cfOptions) { opt.close(); @@ -332,9 +412,6 @@ public synchronized boolean shutdown() { if (this.totalOrderReadOptions != null) { this.totalOrderReadOptions.close(); } - if (this.options != null) { - this.options.close(); - } //4. close db. if (db != null && !this.readOnly) { this.db.syncWal(); @@ -342,6 +419,10 @@ public synchronized boolean shutdown() { if (db != null) { this.db.closeE(); } + // Close DBOptions after RocksDB instance is closed. + if (this.options != null) { + this.options.close(); + } //5. help gc. this.cfOptions.clear(); this.db = null; @@ -360,21 +441,33 @@ public synchronized boolean shutdown() { return true; } - public void flush(final FlushOptions flushOptions) { + public void flush(final FlushOptions flushOptions) throws RocksDBException { + flush(flushOptions, this.cfHandles); + } + + public void flush(final FlushOptions flushOptions, List columnFamilyHandles) throws RocksDBException { if (!this.loaded || this.readOnly || closed) { return; } try { if (db != null) { - this.db.flush(flushOptions); + // For atomic-flush, we have to explicitly specify column family handles + // See https://github.com/rust-rocksdb/rust-rocksdb/pull/793 + // and https://github.com/facebook/rocksdb/blob/8ad4c7efc48d301f5e85467105d7019a49984dc8/include/rocksdb/db.h#L1667 + this.db.flush(flushOptions, columnFamilyHandles); } } catch (RocksDBException e) { scheduleReloadRocksdb(e); LOGGER.error("flush Failed. {}, {}", this.dbPath, getStatusError(e)); + throw e; } } + public void flushWAL() throws RocksDBException { + this.db.flushWal(true); + } + public Statistics getStatistics() { return this.options.statistics(); } @@ -441,10 +534,6 @@ private void reloadRocksdb() throws Exception { LOGGER.info("reload rocksdb OK. {}", this.dbPath); } - public void flushWAL() throws RocksDBException { - this.db.flushWal(true); - } - private String getStatusError(RocksDBException e) { if (e == null || e.getStatus() == null) { return "null"; @@ -477,13 +566,13 @@ public void statRocksdb(Logger logger) { Map map = Maps.newHashMap(); for (LiveFileMetaData metaData : liveFileMetaDataList) { StringBuilder sb = map.computeIfAbsent(metaData.level(), k -> new StringBuilder(256)); - sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). - append(metaData.fileName()).append(SPACE). - append("s: ").append(metaData.size()).append(SPACE). - append("a: ").append(metaData.numEntries()).append(SPACE). - append("r: ").append(metaData.numReadsSampled()).append(SPACE). - append("d: ").append(metaData.numDeletions()).append(SPACE). - append(metaData.beingCompacted()).append("\n"); + sb.append(new String(metaData.columnFamilyName(), StandardCharsets.UTF_8)).append(SPACE). + append(metaData.fileName()).append(SPACE). + append("s: ").append(metaData.size()).append(SPACE). + append("a: ").append(metaData.numEntries()).append(SPACE). + append("r: ").append(metaData.numReadsSampled()).append(SPACE). + append("d: ").append(metaData.numDeletions()).append(SPACE). + append(metaData.beingCompacted()).append("\n"); } map.forEach((key, value) -> logger.info("level: {}\n{}", key, value.toString())); @@ -492,11 +581,9 @@ public void statRocksdb(Logger logger) { String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem"); String memTableMemUsage = this.db.getProperty("rocksdb.cur-size-all-mem-tables"); String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage"); - logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, memtable: {}, blocksPinnedByIterator: {}", - blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); - } catch (Exception e) { - logger.error("statRocksdb Failed. {}", this.dbPath, e); - throw new RuntimeException(e); + logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, MemTable: {}, blocksPinnedByIterator: {}", + blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); + } catch (Exception ignored) { } } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java new file mode 100644 index 00000000000..95d5119cfc6 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.config; + +import java.io.File; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.UtilAll; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompactionStyle; +import org.rocksdb.CompressionType; +import org.rocksdb.DBOptions; +import org.rocksdb.DataBlockIndexType; +import org.rocksdb.IndexType; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.LRUCache; +import org.rocksdb.RateLimiter; +import org.rocksdb.SkipListMemTableConfig; +import org.rocksdb.Statistics; +import org.rocksdb.StatsLevel; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.WALRecoveryMode; +import org.rocksdb.util.SizeUnit; + +public class ConfigHelper { + public static ColumnFamilyOptions createConfigOptions() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). + setFormatVersion(5). + setIndexType(IndexType.kBinarySearch). + setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). + setBlockSize(32 * SizeUnit.KB). + setFilterPolicy(new BloomFilter(16, false)). + // Indicating if we'd put index/filter blocks to the block cache. + setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocksWithHighPriority(true). + setPinL0FilterAndIndexBlocksInCache(false). + setPinTopLevelIndexAndFilter(true). + setBlockCache(new LRUCache(4 * SizeUnit.MB, 8, false)). + setWholeKeyFiltering(true); + + ColumnFamilyOptions options = new ColumnFamilyOptions(); + return options.setMaxWriteBufferNumber(2). + // MemTable size, MemTable(cache) -> immutable MemTable(cache) -> SST(disk) + setWriteBufferSize(8 * SizeUnit.MB). + setMinWriteBufferNumberToMerge(1). + setTableFormatConfig(blockBasedTableConfig). + setMemTableConfig(new SkipListMemTableConfig()). + setCompressionType(CompressionType.NO_COMPRESSION). + setNumLevels(7). + setCompactionStyle(CompactionStyle.LEVEL). + setLevel0FileNumCompactionTrigger(4). + setLevel0SlowdownWritesTrigger(8). + setLevel0StopWritesTrigger(12). + // The target file size for compaction. + setTargetFileSizeBase(64 * SizeUnit.MB). + setTargetFileSizeMultiplier(2). + // The upper-bound of the total size of L1 files in bytes + setMaxBytesForLevelBase(256 * SizeUnit.MB). + setMaxBytesForLevelMultiplier(2). + setMergeOperator(new StringAppendOperator()). + setInplaceUpdateSupport(true); + } + + public static DBOptions createConfigDBOptions() { + //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java + DBOptions options = new DBOptions(); + Statistics statistics = new Statistics(); + statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); + return options. + setDbLogDir(getDBLogDir()). + setInfoLogLevel(InfoLogLevel.INFO_LEVEL). + setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords). + setManualWalFlush(true). + setMaxTotalWalSize(500 * SizeUnit.MB). + setWalSizeLimitMB(0). + setWalTtlSeconds(0). + setCreateIfMissing(true). + setCreateMissingColumnFamilies(true). + setMaxOpenFiles(-1). + setMaxLogFileSize(SizeUnit.GB). + setKeepLogFileNum(5). + setMaxManifestFileSize(SizeUnit.GB). + setAllowConcurrentMemtableWrite(false). + setStatistics(statistics). + setStatsDumpPeriodSec(600). + setAtomicFlush(true). + setMaxBackgroundJobs(32). + setMaxSubcompactions(4). + setParanoidChecks(true). + setDelayedWriteRate(16 * SizeUnit.MB). + setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). + setUseDirectIoForFlushAndCompaction(true). + setUseDirectReads(true); + } + + public static String getDBLogDir() { + String rootPath = System.getProperty("user.home"); + if (StringUtils.isEmpty(rootPath)) { + return ""; + } + rootPath = rootPath + File.separator + "logs"; + UtilAll.ensureDirOK(rootPath); + return rootPath + File.separator + "rocketmqlogs" + File.separator; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index f657d9cf2d2..36da6834ff3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -16,101 +16,43 @@ */ package org.apache.rocketmq.common.config; -import java.io.File; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.CompactRangeOptions; -import org.rocksdb.CompactRangeOptions.BottommostLevelCompaction; -import org.rocksdb.CompactionOptions; -import org.rocksdb.CompactionStyle; -import org.rocksdb.CompressionType; -import org.rocksdb.DBOptions; -import org.rocksdb.DataBlockIndexType; -import org.rocksdb.IndexType; -import org.rocksdb.InfoLogLevel; -import org.rocksdb.LRUCache; -import org.rocksdb.RateLimiter; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.SkipListMemTableConfig; -import org.rocksdb.Statistics; -import org.rocksdb.StatsLevel; -import org.rocksdb.StringAppendOperator; -import org.rocksdb.WALRecoveryMode; import org.rocksdb.WriteBatch; -import org.rocksdb.WriteOptions; -import org.rocksdb.util.SizeUnit; public class ConfigRocksDBStorage extends AbstractRocksDBStorage { + public static final byte[] KV_DATA_VERSION_COLUMN_FAMILY_NAME = "kvDataVersion".getBytes(StandardCharsets.UTF_8); + public static final byte[] FORBIDDEN_COLUMN_FAMILY_NAME = "forbidden".getBytes(StandardCharsets.UTF_8); - private static final byte[] KV_DATA_VERSION_COLUMN_FAMILY_NAME = "kvDataVersion".getBytes(StandardCharsets.UTF_8); - private static final byte[] KV_DATA_VERSION_KEY = "kvDataVersionKey".getBytes(StandardCharsets.UTF_8); protected ColumnFamilyHandle kvDataVersionFamilyHandle; - - private static final byte[] FORBIDDEN_COLUMN_FAMILY_NAME = "forbidden".getBytes(StandardCharsets.UTF_8); protected ColumnFamilyHandle forbiddenFamilyHandle; - + public static final byte[] KV_DATA_VERSION_KEY = "kvDataVersionKey".getBytes(StandardCharsets.UTF_8); public ConfigRocksDBStorage(final String dbPath) { - super(); - this.dbPath = dbPath; + super(dbPath); this.readOnly = false; } public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { - super(); - this.dbPath = dbPath; + super(dbPath); this.readOnly = readOnly; } - private void initOptions() { - this.options = createConfigDBOptions(); - - this.writeOptions = new WriteOptions(); - this.writeOptions.setSync(false); - this.writeOptions.setDisableWAL(true); - this.writeOptions.setNoSlowdown(true); - - this.ableWalWriteOptions = new WriteOptions(); - this.ableWalWriteOptions.setSync(false); - this.ableWalWriteOptions.setDisableWAL(false); - this.ableWalWriteOptions.setNoSlowdown(true); - - this.readOptions = new ReadOptions(); - this.readOptions.setPrefixSameAsStart(true); - this.readOptions.setTotalOrderSeek(false); - this.readOptions.setTailing(false); - - this.totalOrderReadOptions = new ReadOptions(); - this.totalOrderReadOptions.setPrefixSameAsStart(false); - this.totalOrderReadOptions.setTotalOrderSeek(false); - this.totalOrderReadOptions.setTailing(false); - - this.compactRangeOptions = new CompactRangeOptions(); - this.compactRangeOptions.setBottommostLevelCompaction(BottommostLevelCompaction.kForce); - this.compactRangeOptions.setAllowWriteStall(true); - this.compactRangeOptions.setExclusiveManualCompaction(false); - this.compactRangeOptions.setChangeLevel(true); - this.compactRangeOptions.setTargetLevel(-1); - this.compactRangeOptions.setMaxSubcompactions(4); - - this.compactionOptions = new CompactionOptions(); - this.compactionOptions.setCompression(CompressionType.LZ4_COMPRESSION); - this.compactionOptions.setMaxSubcompactions(4); - this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L); + protected void initOptions() { + this.options = ConfigHelper.createConfigDBOptions(); + super.initOptions(); } @Override @@ -120,15 +62,14 @@ protected boolean postLoad() { initOptions(); - final List cfDescriptors = new ArrayList(); + final List cfDescriptors = new ArrayList<>(); - ColumnFamilyOptions defaultOptions = createConfigOptions(); + ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigOptions(); this.cfOptions.add(defaultOptions); cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); cfDescriptors.add(new ColumnFamilyDescriptor(KV_DATA_VERSION_COLUMN_FAMILY_NAME, defaultOptions)); cfDescriptors.add(new ColumnFamilyDescriptor(FORBIDDEN_COLUMN_FAMILY_NAME, defaultOptions)); - final List cfHandles = new ArrayList(); - open(cfDescriptors, cfHandles); + open(cfDescriptors); this.defaultCFHandle = cfHandles.get(0); this.kvDataVersionFamilyHandle = cfHandles.get(1); @@ -147,87 +88,6 @@ protected void preShutdown() { this.forbiddenFamilyHandle.close(); } - private ColumnFamilyOptions createConfigOptions() { - BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). - setFormatVersion(5). - setIndexType(IndexType.kBinarySearch). - setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). - setBlockSize(32 * SizeUnit.KB). - setFilterPolicy(new BloomFilter(16, false)). - // Indicating if we'd put index/filter blocks to the block cache. - setCacheIndexAndFilterBlocks(false). - setCacheIndexAndFilterBlocksWithHighPriority(true). - setPinL0FilterAndIndexBlocksInCache(false). - setPinTopLevelIndexAndFilter(true). - setBlockCache(new LRUCache(4 * SizeUnit.MB, 8, false)). - setWholeKeyFiltering(true); - - ColumnFamilyOptions options = new ColumnFamilyOptions(); - return options.setMaxWriteBufferNumber(2). - // MemTable size, memtable(cache) -> immutable memtable(cache) -> sst(disk) - setWriteBufferSize(8 * SizeUnit.MB). - setMinWriteBufferNumberToMerge(1). - setTableFormatConfig(blockBasedTableConfig). - setMemTableConfig(new SkipListMemTableConfig()). - setCompressionType(CompressionType.NO_COMPRESSION). - setNumLevels(7). - setCompactionStyle(CompactionStyle.LEVEL). - setLevel0FileNumCompactionTrigger(4). - setLevel0SlowdownWritesTrigger(8). - setLevel0StopWritesTrigger(12). - // The target file size for compaction. - setTargetFileSizeBase(64 * SizeUnit.MB). - setTargetFileSizeMultiplier(2). - // The upper-bound of the total size of L1 files in bytes - setMaxBytesForLevelBase(256 * SizeUnit.MB). - setMaxBytesForLevelMultiplier(2). - setMergeOperator(new StringAppendOperator()). - setInplaceUpdateSupport(true); - } - - private DBOptions createConfigDBOptions() { - //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide - // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java - DBOptions options = new DBOptions(); - Statistics statistics = new Statistics(); - statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); - return options. - setDbLogDir(getDBLogDir()). - setInfoLogLevel(InfoLogLevel.INFO_LEVEL). - setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords). - setManualWalFlush(true). - setMaxTotalWalSize(500 * SizeUnit.MB). - setWalSizeLimitMB(0). - setWalTtlSeconds(0). - setCreateIfMissing(true). - setCreateMissingColumnFamilies(true). - setMaxOpenFiles(-1). - setMaxLogFileSize(1 * SizeUnit.GB). - setKeepLogFileNum(5). - setMaxManifestFileSize(1 * SizeUnit.GB). - setAllowConcurrentMemtableWrite(false). - setStatistics(statistics). - setStatsDumpPeriodSec(600). - setAtomicFlush(true). - setMaxBackgroundJobs(32). - setMaxSubcompactions(4). - setParanoidChecks(true). - setDelayedWriteRate(16 * SizeUnit.MB). - setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). - setUseDirectIoForFlushAndCompaction(true). - setUseDirectReads(true); - } - - public static String getDBLogDir() { - String rootPath = System.getProperty("user.home"); - if (StringUtils.isEmpty(rootPath)) { - return ""; - } - rootPath = rootPath + File.separator + "logs"; - UtilAll.ensureDirOK(rootPath); - return rootPath + File.separator + "rocketmqlogs" + File.separator; - } - public void put(final byte[] keyBytes, final int keyLen, final byte[] valueBytes) throws Exception { put(this.defaultCFHandle, this.ableWalWriteOptions, keyBytes, keyLen, valueBytes, valueBytes.length); } @@ -281,10 +141,6 @@ public RocksIterator forbiddenIterator() { return this.db.newIterator(this.forbiddenFamilyHandle, this.totalOrderReadOptions); } - public void rangeDelete(final byte[] startKey, final byte[] endKey) throws RocksDBException { - rangeDelete(this.defaultCFHandle, this.writeOptions, startKey, endKey); - } - public RocksIterator iterator(ReadOptions readOptions) { return this.db.newIterator(this.defaultCFHandle, readOptions); } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index c889ae7ca85..17b845d8176 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -81,15 +81,15 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir()); - this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4); + this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath); this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); this.writeBatch = new WriteBatch(); this.batchSize = messageStoreConfig.getBatchWriteKvCqSize(); - this.bufferDRList = new ArrayList(batchSize); - this.cqBBPairList = new ArrayList(batchSize); - this.offsetBBPairList = new ArrayList(batchSize); + this.bufferDRList = new ArrayList<>(batchSize); + this.cqBBPairList = new ArrayList<>(batchSize); + this.offsetBBPairList = new ArrayList<>(batchSize); for (int i = 0; i < batchSize; i++) { this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java index 362684560c8..b343a5b4b50 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java @@ -16,53 +16,45 @@ */ package org.apache.rocketmq.store.rocksdb; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.config.AbstractRocksDBStorage; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.store.MessageStore; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.CompactRangeOptions; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; -import org.rocksdb.WriteOptions; public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage { + + public static final byte[] OFFSET_COLUMN_FAMILY = "offset".getBytes(StandardCharsets.UTF_8); + private final MessageStore messageStore; private volatile ColumnFamilyHandle offsetCFHandle; - public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) { + public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath) { + super(dbPath); this.messageStore = messageStore; - this.dbPath = dbPath; this.readOnly = false; } - private void initOptions() { + protected void initOptions() { this.options = RocksDBOptionsFactory.createDBOptions(); + super.initOptions(); + } - this.writeOptions = new WriteOptions(); - this.writeOptions.setSync(false); - this.writeOptions.setDisableWAL(true); - this.writeOptions.setNoSlowdown(true); - + @Override + protected void initTotalOrderReadOptions() { this.totalOrderReadOptions = new ReadOptions(); this.totalOrderReadOptions.setPrefixSameAsStart(false); this.totalOrderReadOptions.setTotalOrderSeek(false); - - this.compactRangeOptions = new CompactRangeOptions(); - this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); - this.compactRangeOptions.setAllowWriteStall(true); - this.compactRangeOptions.setExclusiveManualCompaction(false); - this.compactRangeOptions.setChangeLevel(true); - this.compactRangeOptions.setTargetLevel(-1); - this.compactRangeOptions.setMaxSubcompactions(4); } @Override @@ -72,7 +64,7 @@ protected boolean postLoad() { initOptions(); - final List cfDescriptors = new ArrayList(); + final List cfDescriptors = new ArrayList<>(); ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore); this.cfOptions.add(cqCfOptions); @@ -80,11 +72,8 @@ protected boolean postLoad() { ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions(); this.cfOptions.add(offsetCfOptions); - cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions)); - - final List cfHandles = new ArrayList(); - open(cfDescriptors, cfHandles); - + cfDescriptors.add(new ColumnFamilyDescriptor(OFFSET_COLUMN_FAMILY, offsetCfOptions)); + open(cfDescriptors); this.defaultCFHandle = cfHandles.get(0); this.offsetCFHandle = cfHandles.get(1); } catch (final Exception e) { @@ -130,4 +119,4 @@ public RocksIterator seekOffsetCF() { public ColumnFamilyHandle getOffsetCFHandle() { return this.offsetCFHandle; } -} \ No newline at end of file +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index a3a99d3346c..c7d5041bd8c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -16,7 +16,7 @@ */ package org.apache.rocketmq.store.rocksdb; -import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +import org.apache.rocketmq.common.config.ConfigHelper; import org.apache.rocketmq.store.MessageStore; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; @@ -71,7 +71,7 @@ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageSt setTableFormatConfig(blockBasedTableConfig). setMemTableConfig(new SkipListMemTableConfig()). setCompressionType(CompressionType.LZ4_COMPRESSION). - setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION). + setBottommostCompressionType(CompressionType.LZ4_COMPRESSION). setNumLevels(7). setCompactionStyle(CompactionStyle.UNIVERSAL). setCompactionOptionsUniversal(compactionOption). @@ -134,7 +134,7 @@ public static DBOptions createDBOptions() { Statistics statistics = new Statistics(); statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); return options. - setDbLogDir(ConfigRocksDBStorage.getDBLogDir()). + setDbLogDir(ConfigHelper.getDBLogDir()). setInfoLogLevel(InfoLogLevel.INFO_LEVEL). setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). setManualWalFlush(true). @@ -144,9 +144,9 @@ public static DBOptions createDBOptions() { setCreateIfMissing(true). setCreateMissingColumnFamilies(true). setMaxOpenFiles(-1). - setMaxLogFileSize(1 * SizeUnit.GB). + setMaxLogFileSize(SizeUnit.GB). setKeepLogFileNum(5). - setMaxManifestFileSize(1 * SizeUnit.GB). + setMaxManifestFileSize(SizeUnit.GB). setAllowConcurrentMemtableWrite(false). setStatistics(statistics). setAtomicFlush(true). From 1e0b08f56ab628c9017536eba8509d8987ec6be5 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Fri, 18 Oct 2024 16:50:13 +0800 Subject: [PATCH 191/265] fix: make ConsumeQueueStore bottom most compression type configurable (#8841) Signed-off-by: Li Zhanhui --- .../store/config/MessageStoreConfig.java | 23 +++++++++++++ .../store/rocksdb/RocksDBOptionsFactory.java | 5 ++- .../rocksdb/RocksDBOptionsFactoryTest.java | 34 +++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 68531284389..5195868e0f1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -430,6 +430,22 @@ public class MessageStoreConfig { private int batchWriteKvCqSize = 16; + /** + * If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type. + * The following values are valid: + *

      + *
    • snappy
      • + *
    • z
      • + *
    • bzip2
      • + *
    • lz4
      • + *
    • lz4hc
      • + *
    • xpress
      • + *
    • zstd
      • + *
    + * + * LZ4 is the recommended one. + */ + private String bottomMostCompressionTypeForConsumeQueueStore = "zstd"; public int getBatchWriteKvCqSize() { return batchWriteKvCqSize; @@ -1885,4 +1901,11 @@ public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocks this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; } + public String getBottomMostCompressionTypeForConsumeQueueStore() { + return bottomMostCompressionTypeForConsumeQueueStore; + } + + public void setBottomMostCompressionTypeForConsumeQueueStore(String bottomMostCompressionTypeForConsumeQueueStore) { + this.bottomMostCompressionTypeForConsumeQueueStore = bottomMostCompressionTypeForConsumeQueueStore; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index c7d5041bd8c..d373ba6249c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -65,13 +65,16 @@ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageSt setMaxMergeWidth(Integer.MAX_VALUE). setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize). setCompressionSizePercent(-1); + String bottomMostCompressionTypeOpt = messageStore.getMessageStoreConfig() + .getBottomMostCompressionTypeForConsumeQueueStore(); + CompressionType bottomMostCompressionType = CompressionType.getCompressionType(bottomMostCompressionTypeOpt); return columnFamilyOptions.setMaxWriteBufferNumber(4). setWriteBufferSize(128 * SizeUnit.MB). setMinWriteBufferNumberToMerge(1). setTableFormatConfig(blockBasedTableConfig). setMemTableConfig(new SkipListMemTableConfig()). setCompressionType(CompressionType.LZ4_COMPRESSION). - setBottommostCompressionType(CompressionType.LZ4_COMPRESSION). + setBottommostCompressionType(bottomMostCompressionType). setNumLevels(7). setCompactionStyle(CompactionStyle.UNIVERSAL). setCompactionOptionsUniversal(compactionOption). diff --git a/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java b/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java new file mode 100644 index 00000000000..1d7273968f6 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactoryTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.rocksdb; + +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Assert; +import org.junit.Test; +import org.rocksdb.CompressionType; + +public class RocksDBOptionsFactoryTest { + + @Test + public void testBottomMostCompressionType() { + MessageStoreConfig config = new MessageStoreConfig(); + Assert.assertEquals(CompressionType.ZSTD_COMPRESSION, + CompressionType.getCompressionType(config.getBottomMostCompressionTypeForConsumeQueueStore())); + Assert.assertEquals(CompressionType.LZ4_COMPRESSION, CompressionType.getCompressionType("lz4")); + } +} From 07f13fd883e87e40d7de4827e28913e617fb9832 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Mon, 21 Oct 2024 09:25:55 +0800 Subject: [PATCH 192/265] fix: remove unnecessary synchronized to improve concurrency (#8840) Signed-off-by: Li Zhanhui --- .../broker/client/ProducerManager.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index f9fe1193e22..011c9e4be3c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; @@ -39,11 +40,11 @@ public class ProducerManager { private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private static final int GET_AVAILABLE_CHANNEL_RETRY_COUNT = 3; - private final ConcurrentHashMap> groupChannelTable = + private final ConcurrentMap> groupChannelTable = new ConcurrentHashMap<>(); - private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); + private final ConcurrentMap clientChannelTable = new ConcurrentHashMap<>(); protected final BrokerStatsManager brokerStatsManager; - private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); + private final PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); private final List producerChangeListenerList = new CopyOnWriteArrayList<>(); public ProducerManager() { @@ -63,7 +64,7 @@ public boolean groupOnline(String group) { return channels != null && !channels.isEmpty(); } - public ConcurrentHashMap> getGroupChannelTable() { + public ConcurrentMap> getGroupChannelTable() { return groupChannelTable; } @@ -95,13 +96,13 @@ public ProducerTableInfo getProducerTable() { } public void scanNotActiveChannel() { - Iterator>> iterator = this.groupChannelTable.entrySet().iterator(); + Iterator>> iterator = this.groupChannelTable.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry> entry = iterator.next(); + Map.Entry> entry = iterator.next(); final String group = entry.getKey(); - final ConcurrentHashMap chlMap = entry.getValue(); + final ConcurrentMap chlMap = entry.getValue(); Iterator> it = chlMap.entrySet().iterator(); while (it.hasNext()) { @@ -132,16 +133,13 @@ public void scanNotActiveChannel() { } } - public synchronized boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) { + public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) { boolean removed = false; if (channel != null) { - for (final Map.Entry> entry : this.groupChannelTable - .entrySet()) { + for (final Map.Entry> entry : this.groupChannelTable.entrySet()) { final String group = entry.getKey(); - final ConcurrentHashMap clientChannelInfoTable = - entry.getValue(); - final ClientChannelInfo clientChannelInfo = - clientChannelInfoTable.remove(channel); + final ConcurrentMap clientChannelInfoTable = entry.getValue(); + final ClientChannelInfo clientChannelInfo = clientChannelInfoTable.remove(channel); if (clientChannelInfo != null) { clientChannelTable.remove(clientChannelInfo.getClientId()); removed = true; @@ -150,7 +148,7 @@ public synchronized boolean doChannelCloseEvent(final String remoteAddr, final C clientChannelInfo.toString(), remoteAddr, group); callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo); if (clientChannelInfoTable.isEmpty()) { - ConcurrentHashMap oldGroupTable = this.groupChannelTable.remove(group); + ConcurrentMap oldGroupTable = this.groupChannelTable.remove(group); if (oldGroupTable != null) { log.info("unregister a producer group[{}] from groupChannelTable", group); callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null); @@ -163,13 +161,16 @@ public synchronized boolean doChannelCloseEvent(final String remoteAddr, final C return removed; } - public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { - ClientChannelInfo clientChannelInfoFound = null; + public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { + ClientChannelInfo clientChannelInfoFound; - ConcurrentHashMap channelTable = this.groupChannelTable.get(group); + ConcurrentMap channelTable = this.groupChannelTable.get(group); if (null == channelTable) { channelTable = new ConcurrentHashMap<>(); - this.groupChannelTable.put(group, channelTable); + ConcurrentMap prev = this.groupChannelTable.putIfAbsent(group, channelTable); + if (null != prev) { + channelTable = prev; + } } clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); @@ -186,8 +187,8 @@ public synchronized void registerProducer(final String group, final ClientChanne } } - public synchronized void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { - ConcurrentHashMap channelTable = this.groupChannelTable.get(group); + public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { + ConcurrentMap channelTable = this.groupChannelTable.get(group); if (null != channelTable && !channelTable.isEmpty()) { ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); clientChannelTable.remove(clientChannelInfo.getClientId()); @@ -210,7 +211,7 @@ public Channel getAvailableChannel(String groupId) { return null; } List channelList; - ConcurrentHashMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); + ConcurrentMap channelClientChannelInfoHashMap = groupChannelTable.get(groupId); if (channelClientChannelInfoHashMap != null) { channelList = new ArrayList<>(channelClientChannelInfoHashMap.keySet()); } else { From a024dc81dafad21182a8485c1216b97e79b64661 Mon Sep 17 00:00:00 2001 From: dinglei Date: Mon, 21 Oct 2024 13:40:49 +0800 Subject: [PATCH 193/265] update netty version to 4.1.114 to fix CVE-2023-34462 (#8832) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b18d9bbb439..33db3c7f486 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ 1.8 1.5.0 - 4.1.65.Final + 4.1.114.Final 2.0.53.Final 1.69 1.2.83 From c033c3e85f9487e07a0c521082df34f1cd686bb4 Mon Sep 17 00:00:00 2001 From: mawen12 <1181963012mw@gmail.com> Date: Wed, 23 Oct 2024 08:29:35 +0800 Subject: [PATCH 194/265] [ISSUE #8848] Fix log typo --- .../rocketmq/broker/BrokerController.java | 2 +- .../client/impl/factory/MQClientInstance.java | 2 +- .../StatisticsBriefInterceptor.java | 2 +- ...atisticsItemScheduledIncrementPrinter.java | 12 +++++------ .../statictopic/TopicQueueMappingUtils.java | 20 +++++++++---------- .../apache/rocketmq/store/timer/TimerLog.java | 2 +- .../rocketmq/test/util/MQAdminTestUtils.java | 4 ++-- .../broadcast/order/OrderMsgBroadcastIT.java | 2 +- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index aaf06caddf8..05a00a50053 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -1010,7 +1010,7 @@ private void initialTransaction() { private void initialAcl() { if (!this.brokerConfig.isAclEnable()) { - LOG.info("The broker dose not enable acl"); + LOG.info("The broker does not enable acl"); return; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index c9fd3c83e04..ad0676d091c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -1238,7 +1238,7 @@ public synchronized void resetOffset(String topic, String group, Map[] String name = briefMetas[i].getKey(); int index = ArrayUtils.indexOf(item.getItemNames(), name); if (index < 0) { - throw new IllegalArgumentException("illegal breifItemName: " + name); + throw new IllegalArgumentException("illegal briefItemName: " + name); } indexOfItems[i] = index; statisticsBriefs[i] = new StatisticsBrief(briefMetas[i].getValue()); diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java index 2890e6e15cd..e1998473bf2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java @@ -65,15 +65,15 @@ public void run() { item.getStatObject()); StatisticsItem increment = snapshot.subtract(lastSnapshot); - Interceptor inteceptor = item.getInterceptor(); - String inteceptorStr = formatInterceptor(inteceptor); - if (inteceptor != null) { - inteceptor.reset(); + Interceptor interceptor = item.getInterceptor(); + String interceptorStr = formatInterceptor(interceptor); + if (interceptor != null) { + interceptor.reset(); } StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject()); if (brief != null && (!increment.allZeros() || printZeroLine())) { - printer.print(name, increment, inteceptorStr, brief.toString()); + printer.print(name, increment, interceptorStr, brief.toString()); } setItemSnapshot(lastItemSnapshots, snapshot); @@ -85,7 +85,7 @@ public void run() { }, getInitialDelay(), interval, TimeUnit.MILLISECONDS); addFuture(item, future); - // sample every TPS_INTREVAL + // sample every TPS_INTERVAL ScheduledFuture futureSample = executor.scheduleAtFixedRate(new Runnable() { @Override public void run() { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java index 45cbed75727..647669fde24 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicQueueMappingUtils.java @@ -172,21 +172,21 @@ public static Map.Entry checkNameEpochNumConsistence(String topic if (scope != null && !scope.equals(mappingDetail.getScope())) { - throw new RuntimeException(String.format("scope dose not match %s != %s in %s", mappingDetail.getScope(), scope, broker)); + throw new RuntimeException(String.format("scope does not match %s != %s in %s", mappingDetail.getScope(), scope, broker)); } else { scope = mappingDetail.getScope(); } if (maxEpoch != -1 && maxEpoch != mappingDetail.getEpoch()) { - throw new RuntimeException(String.format("epoch dose not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname())); + throw new RuntimeException(String.format("epoch does not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname())); } else { maxEpoch = mappingDetail.getEpoch(); } if (maxNum != -1 && maxNum != mappingDetail.getTotalQueues()) { - throw new RuntimeException(String.format("total queue number dose not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname())); + throw new RuntimeException(String.format("total queue number does not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname())); } else { maxNum = mappingDetail.getTotalQueues(); } @@ -266,7 +266,7 @@ public static void checkLogicQueueMappingItemOffset(List throw new RuntimeException("The non-latest item has negative logic offset"); } if (lastGen != -1 && item.getGen() >= lastGen) { - throw new RuntimeException("The gen dose not increase monotonically"); + throw new RuntimeException("The gen does not increase monotonically"); } if (item.getEndOffset() != -1 @@ -276,10 +276,10 @@ public static void checkLogicQueueMappingItemOffset(List if (lastOffset != -1 && item.getLogicOffset() != -1) { if (item.getLogicOffset() >= lastOffset) { - throw new RuntimeException("The base logic offset dose not increase monotonically"); + throw new RuntimeException("The base logic offset does not increase monotonically"); } if (item.computeMaxStaticQueueOffset() >= lastOffset) { - throw new RuntimeException("The max logic offset dose not increase monotonically"); + throw new RuntimeException("The max logic offset does not increase monotonically"); } } lastGen = item.getGen(); @@ -321,11 +321,11 @@ public static void checkPhysicalQueueConsistence(Map items: configMapping.getMappingDetail().getHostedQueues().values()) { for (LogicQueueMappingItem item: items) { if (item.getStartOffset() != 0) { - throw new RuntimeException("The start offset dose not begin from 0"); + throw new RuntimeException("The start offset does not begin from 0"); } TopicConfig topicConfig = brokerConfigMap.get(item.getBname()); if (topicConfig == null) { - throw new RuntimeException("The broker of item dose not exist"); + throw new RuntimeException("The broker of item does not exist"); } if (item.getQueueId() >= topicConfig.getWriteQueueNums()) { throw new RuntimeException("The physical queue id is overflow the write queues"); @@ -365,7 +365,7 @@ public static Map checkAndBuildMappingItems(List< } if (checkConsistence) { if (maxNum != globalIdMap.size()) { - throw new RuntimeException(String.format("The total queue number in config dose not match the real hosted queues %d != %d", maxNum, globalIdMap.size())); + throw new RuntimeException(String.format("The total queue number in config does not match the real hosted queues %d != %d", maxNum, globalIdMap.size())); } for (int i = 0; i < maxNum; i++) { if (!globalIdMap.containsKey(i)) { @@ -417,7 +417,7 @@ public static void checkTargetBrokersComplete(Set targetBrokers, Map createStaticTopic(String topic, int queueNum, Set targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception { Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); assert brokerConfigMap.isEmpty(); @@ -203,7 +203,7 @@ public static Map createStaticTopic(String t return brokerConfigMap; } - //should only be test, if some middle operation failed, it dose not backup the brokerConfigMap + //should only be test, if some middle operation failed, it does not backup the brokerConfigMap public static void remappingStaticTopic(String topic, Set targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception { Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); assert !brokerConfigMap.isEmpty(); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java index 679fdd4f381..d1e2c0ddaf3 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadcastIT.java @@ -36,7 +36,7 @@ import static com.google.common.truth.Truth.assertThat; /** - * Currently, dose not support the ordered broadcast message + * Currently, does not support the ordered broadcast message */ @Ignore public class OrderMsgBroadcastIT extends BaseBroadcast { From d2fd068be77d06495d810b799d29c2d1f222e4dc Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 23 Oct 2024 09:37:17 +0800 Subject: [PATCH 195/265] =?UTF-8?q?fix:=20registerProducer=20should=20not?= =?UTF-8?q?=20be=20affected=20by=20concurrent=20scanNotAct=E2=80=A6=20(#88?= =?UTF-8?q?47)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: registerProducer should not be affected by concurrent scanNotActiveChannel Signed-off-by: Li Zhanhui * chore: fix code format and make CI pass Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui --- .../broker/client/ProducerManager.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 011c9e4be3c..2c3acb6ba9b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -71,15 +71,15 @@ public ConcurrentMap> getGroup public ProducerTableInfo getProducerTable() { Map> map = new HashMap<>(); for (String group : this.groupChannelTable.keySet()) { - for (Entry entry: this.groupChannelTable.get(group).entrySet()) { + for (Entry entry : this.groupChannelTable.get(group).entrySet()) { ClientChannelInfo clientChannelInfo = entry.getValue(); if (map.containsKey(group)) { map.get(group).add(new ProducerInfo( - clientChannelInfo.getClientId(), - clientChannelInfo.getChannel().remoteAddress().toString(), - clientChannelInfo.getLanguage(), - clientChannelInfo.getVersion(), - clientChannelInfo.getLastUpdateTimestamp() + clientChannelInfo.getClientId(), + clientChannelInfo.getChannel().remoteAddress().toString(), + clientChannelInfo.getLanguage(), + clientChannelInfo.getVersion(), + clientChannelInfo.getLastUpdateTimestamp() )); } else { map.put(group, new ArrayList<>(Collections.singleton(new ProducerInfo( @@ -118,8 +118,8 @@ public void scanNotActiveChannel() { clientChannelTable.remove(info.getClientId()); } log.warn( - "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", - RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); + "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", + RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, info); RemotingHelper.closeChannel(info.getChannel()); } @@ -144,8 +144,8 @@ public boolean doChannelCloseEvent(final String remoteAddr, final Channel channe clientChannelTable.remove(clientChannelInfo.getClientId()); removed = true; log.info( - "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", - clientChannelInfo.toString(), remoteAddr, group); + "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", + clientChannelInfo.toString(), remoteAddr, group); callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo); if (clientChannelInfoTable.isEmpty()) { ConcurrentMap oldGroupTable = this.groupChannelTable.remove(group); @@ -167,21 +167,26 @@ public void registerProducer(final String group, final ClientChannelInfo clientC ConcurrentMap channelTable = this.groupChannelTable.get(group); if (null == channelTable) { channelTable = new ConcurrentHashMap<>(); + // Make sure channelTable will NOT be cleaned by #scanNotActiveChannel + channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); ConcurrentMap prev = this.groupChannelTable.putIfAbsent(group, channelTable); - if (null != prev) { + if (null == prev) { + // Add client-id to channel mapping for new producer group + clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel()); + } else { channelTable = prev; } } clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); + // Add client-channel info to existing producer group if (null == clientChannelInfoFound) { channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel()); - log.info("new producer connected, group: {} channel: {}", group, - clientChannelInfo.toString()); + log.info("new producer connected, group: {} channel: {}", group, clientChannelInfo.toString()); } - + // Refresh existing client-channel-info update-timestamp if (clientChannelInfoFound != null) { clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis()); } @@ -193,8 +198,7 @@ public void unregisterProducer(final String group, final ClientChannelInfo clien ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); clientChannelTable.remove(clientChannelInfo.getClientId()); if (old != null) { - log.info("unregister a producer[{}] from groupChannelTable {}", group, - clientChannelInfo.toString()); + log.info("unregister a producer[{}] from groupChannelTable {}", group, clientChannelInfo.toString()); callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo); } From b86059c7c16275d50d216bc32f9da816ed2ddab0 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 23 Oct 2024 09:56:37 +0800 Subject: [PATCH 196/265] Support LMQ dispatch in case if Consume Queue Store is RocksDB-based (#8842) * feat: support LMQ dispatch Signed-off-by: Li Zhanhui * fix: introduce group-commit for batch insertion of RocksDB KV pairs Signed-off-by: Li Zhanhui * fix: propagate store error to broker module Signed-off-by: Li Zhanhui * chore: fix all Bazel warning and errors Signed-off-by: Zhanhui Li * fix: remove unnecessary batch-ops when writing RocksDB using atomic flush Signed-off-by: Li Zhanhui * fix: find a writable directory for RocksDB logs Signed-off-by: Li Zhanhui * chore: clean up ConfigHelperTest Signed-off-by: Li Zhanhui * fix: truncate consume queues in case commit log records are truncated Signed-off-by: Li Zhanhui * fix: truncate LMQ max offsets Signed-off-by: Li Zhanhui * fix: correct truncate boundary of consume queues Signed-off-by: Li Zhanhui * fix: correct MessageExt encoding Signed-off-by: Li Zhanhui * chore: remove unused import Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui Signed-off-by: Zhanhui Li --- .gitignore | 3 +- MODULE.bazel | 22 ++ WORKSPACE | 2 + broker/BUILD.bazel | 3 + .../broker/client/net/Broker2Client.java | 14 +- .../LmqPullRequestHoldService.java | 3 +- .../longpolling/PullRequestHoldService.java | 10 +- .../broker/metrics/ConsumerLagCalculator.java | 98 +++--- .../broker/metrics/PopMetricsManager.java | 24 +- .../broker/offset/BroadcastOffsetManager.java | 34 +- .../broker/processor/AckMessageProcessor.java | 21 +- .../processor/AdminBrokerProcessor.java | 292 ++++++++++-------- .../ChangeInvisibleTimeProcessor.java | 14 +- .../processor/NotificationProcessor.java | 17 +- .../processor/PeekMessageProcessor.java | 10 +- .../broker/processor/PopMessageProcessor.java | 55 +++- .../broker/processor/PopReviveService.java | 22 +- .../processor/PullMessageProcessor.java | 33 +- .../schedule/ScheduleMessageService.java | 3 +- .../broker/client/net/Broker2ClientTest.java | 7 +- .../offset/BroadcastOffsetManagerTest.java | 7 +- ...merOrderInfoManagerLockFreeNotifyTest.java | 5 +- .../processor/AckMessageProcessorTest.java | 3 +- .../processor/PopMessageProcessorTest.java | 3 +- client/BUILD.bazel | 1 + .../rocketmq/client/impl/MQClientAPIImpl.java | 8 +- .../client/impl/MQClientAPIImplTest.java | 4 +- .../org/apache/rocketmq/common/MixAll.java | 11 +- .../apache/rocketmq/common/ServiceThread.java | 2 +- .../rocketmq/common/config/ConfigHelper.java | 28 +- .../common/message/MessageExtBrokerInner.java | 10 +- .../common/config/ConfigHelperTest.java | 30 ++ .../rocketmq/example/lmq/LMQProducer.java | 2 +- .../remoting/netty/FileRegionEncoderTest.java | 4 +- store/BUILD.bazel | 3 + .../rocketmq/store/AppendMessageStatus.java | 1 + .../org/apache/rocketmq/store/CommitLog.java | 46 +-- .../apache/rocketmq/store/ConsumeQueue.java | 4 +- .../rocketmq/store/DefaultMessageStore.java | 66 ++-- .../rocketmq/store/DispatchRequest.java | 15 + .../apache/rocketmq/store/LmqDispatch.java | 56 ++++ .../rocketmq/store/MessageExtEncoder.java | 4 +- .../apache/rocketmq/store/MessageStore.java | 15 +- .../apache/rocketmq/store/MultiDispatch.java | 77 ----- .../rocketmq/store/RocksDBMessageStore.java | 9 - .../store/config/MessageStoreConfig.java | 10 - .../exception/ConsumeQueueException.java | 39 +++ .../store/exception/StoreException.java | 38 +++ .../plugin/AbstractPluginMessageStore.java | 12 +- .../queue/AbstractConsumeQueueStore.java | 17 +- .../store/queue/ConsumeQueueStore.java | 16 + .../queue/ConsumeQueueStoreInterface.java | 39 ++- .../rocketmq/store/queue/DispatchEntry.java | 47 +++ .../store/queue/OffsetInitializer.java | 23 ++ .../queue/OffsetInitializerRocksDBImpl.java | 44 +++ .../store/queue/QueueOffsetOperator.java | 39 ++- .../queue/RocksDBConsumeQueueOffsetTable.java | 286 +++++++++++------ .../store/queue/RocksDBConsumeQueueStore.java | 224 ++++++++++---- .../store/queue/RocksDBConsumeQueueTable.java | 31 +- .../store/queue/offset/OffsetEntry.java | 45 +++ .../store/queue/offset/OffsetEntryType.java | 23 ++ .../store/DefaultMessageStoreTest.java | 5 +- .../rocketmq/store/MultiDispatchTest.java | 105 ------- .../store/RocksDBMessageStoreTest.java | 5 +- .../test/offset/LagCalculationIT.java | 5 +- .../core/MessageStoreDispatcherImpl.java | 5 + .../metrics/TieredStoreMetricsManager.java | 91 +++--- tools/BUILD.bazel | 4 +- 68 files changed, 1440 insertions(+), 814 deletions(-) create mode 100644 MODULE.bazel create mode 100644 common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java delete mode 100644 store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java delete mode 100644 store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java diff --git a/.gitignore b/.gitignore index c20f4bf7685..4ee76210738 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ bazel-out bazel-bin bazel-rocketmq bazel-testlogs -.vscode \ No newline at end of file +.vscode +MODULE.bazel.lock \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 00000000000..15fc5c6e3a6 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/WORKSPACE b/WORKSPACE index e1f7743302a..9b06bc63413 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -112,6 +112,8 @@ maven_install( "com.alipay.sofa:hessian:3.3.6", "io.netty:netty-tcnative-boringssl-static:2.0.48.Final", "org.mockito:mockito-junit-jupiter:4.11.0", + "com.alibaba.fastjson2:fastjson2:2.0.43", + "org.junit.jupiter:junit-jupiter-api:5.9.1", ], fetch_sources = True, repositories = [ diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 66e621e9301..c21f9d114c3 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -91,6 +91,9 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:org_powermock_powermock_core", "@maven//:io_opentelemetry_opentelemetry_api", + "@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:commons_collections_commons_collections", ], ) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index 8d95d843dba..f43f79b1be2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -49,6 +50,7 @@ import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class Broker2Client { private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -100,13 +102,12 @@ public void notifyConsumerIdsChanged( } } - - public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) { + public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) throws RemotingCommandException { return resetOffset(topic, group, timeStamp, isForce, false); } public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, - boolean isC) { + boolean isC) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -135,8 +136,11 @@ public RemotingCommand resetOffset(String topic, String group, long timeStamp, b long timeStampOffset; if (timeStamp == -1) { - - timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + try { + timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } } else { timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java index 88e74fd6e5a..eddaee706a9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java @@ -22,7 +22,6 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; - public class LmqPullRequestHoldService extends PullRequestHoldService { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -48,8 +47,8 @@ public void checkHoldRequest() { } String topic = key.substring(0, idx); int queueId = Integer.parseInt(key.substring(idx + 1)); - final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); try { + final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); this.notifyMessageArriving(topic, queueId, offset); } catch (Throwable e) { LOGGER.error("check hold request failed. topic={}, queueId={}", topic, queueId, e); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java index e8da9d0c47c..7dbc9e4fd86 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java @@ -28,6 +28,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.ConsumeQueueExt; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class PullRequestHoldService extends ServiceThread { private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -103,8 +104,8 @@ protected void checkHoldRequest() { if (2 == kArray.length) { String topic = kArray[0]; int queueId = Integer.parseInt(kArray[1]); - final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); try { + final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); this.notifyMessageArriving(topic, queueId, offset); } catch (Throwable e) { log.error( @@ -131,7 +132,12 @@ public void notifyMessageArriving(final String topic, final int queueId, final l for (PullRequest request : requestList) { long newestOffset = maxOffset; if (newestOffset <= request.getPullFromThisOffset()) { - newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + try { + newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + } catch (ConsumeQueueException e) { + log.error("Failed tp get max offset in queue", e); + continue; + } } if (newestOffset > request.getPullFromThisOffset()) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java index 1930d0dfcb6..3ac6528b2a4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.store.DefaultMessageFilter; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class ConsumerLagCalculator { private final BrokerConfig brokerConfig; @@ -212,22 +213,30 @@ public void calculateLag(Consumer lagRecorder) { CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false); - Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); - if (lag != null) { - result.lag = lag.getObject1(); - result.earliestUnconsumedTimestamp = lag.getObject2(); + try { + Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); + if (lag != null) { + result.lag = lag.getObject1(); + result.earliestUnconsumedTimestamp = lag.getObject2(); + } + lagRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get lag stats", e); } - lagRecorder.accept(result); if (info.isPop) { - Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); + try { + Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); - result = new CalculateLagResult(info.group, info.topic, true); - if (retryLag != null) { - result.lag = retryLag.getObject1(); - result.earliestUnconsumedTimestamp = retryLag.getObject2(); + result = new CalculateLagResult(info.group, info.topic, true); + if (retryLag != null) { + result.lag = retryLag.getObject1(); + result.earliestUnconsumedTimestamp = retryLag.getObject2(); + } + lagRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get lag stats", e); } - lagRecorder.accept(result); } }); } @@ -235,22 +244,30 @@ public void calculateLag(Consumer lagRecorder) { public void calculateInflight(Consumer inflightRecorder) { processAllGroup(info -> { CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false); - Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop); - if (inFlight != null) { - result.inFlight = inFlight.getObject1(); - result.earliestUnPulledTimestamp = inFlight.getObject2(); + try { + Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop); + if (inFlight != null) { + result.inFlight = inFlight.getObject1(); + result.earliestUnPulledTimestamp = inFlight.getObject2(); + } + inflightRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get inflight message stats", e); } - inflightRecorder.accept(result); if (info.isPop) { - Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true); + try { + Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true); - result = new CalculateInflightResult(info.group, info.topic, true); - if (retryInFlight != null) { - result.inFlight = retryInFlight.getObject1(); - result.earliestUnPulledTimestamp = retryInFlight.getObject2(); + result = new CalculateInflightResult(info.group, info.topic, true); + if (retryInFlight != null) { + result.inFlight = retryInFlight.getObject1(); + result.earliestUnPulledTimestamp = retryInFlight.getObject2(); + } + inflightRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get inflight message stats", e); } - inflightRecorder.accept(result); } }); } @@ -259,20 +276,28 @@ public void calculateAvailable(Consumer availableRecor processAllGroup(info -> { CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false); - result.available = getAvailableMsgCount(info.group, info.topic, info.isPop); - availableRecorder.accept(result); + try { + result.available = getAvailableMsgCount(info.group, info.topic, info.isPop); + availableRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get available message count", e); + } - if (info.isPop) { - long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true); - result = new CalculateAvailableResult(info.group, info.topic, true); - result.available = retryAvailable; - availableRecorder.accept(result); + if (info.isPop) { + try { + long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true); + result = new CalculateAvailableResult(info.group, info.topic, true); + result.available = retryAvailable; + availableRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get available message count", e); + } } }); } - public Pair getConsumerLagStats(String group, String topic, boolean isPop) { + public Pair getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; long earliestUnconsumedTimestamp = Long.MAX_VALUE; @@ -298,7 +323,8 @@ public Pair getConsumerLagStats(String group, String topic, boolean return new Pair<>(total, earliestUnconsumedTimestamp); } - public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop) { + public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId); if (brokerOffset < 0) { brokerOffset = 0; @@ -329,7 +355,7 @@ public Pair getConsumerLagStats(String group, String topic, int queu return new Pair<>(lag, consumerStoreTimeStamp); } - public Pair getInFlightMsgStats(String group, String topic, boolean isPop) { + public Pair getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; long earliestUnPulledTimestamp = Long.MAX_VALUE; @@ -355,7 +381,8 @@ public Pair getInFlightMsgStats(String group, String topic, boolean return new Pair<>(total, earliestUnPulledTimestamp); } - public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) { + public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { if (isPop) { long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId); long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId); @@ -384,7 +411,7 @@ public Pair getInFlightMsgStats(String group, String topic, int queu return new Pair<>(inflight, pullStoreTimeStamp); } - public long getAvailableMsgCount(String group, String topic, boolean isPop) { + public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; if (group == null || topic == null) { @@ -403,7 +430,8 @@ public long getAvailableMsgCount(String group, String topic, boolean isPop) { return total; } - public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) { + public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId); if (brokerOffset < 0) { brokerOffset = 0; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java index 2de220da166..6e87cb0b69e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java @@ -39,8 +39,11 @@ import org.apache.rocketmq.common.metrics.NopLongCounter; import org.apache.rocketmq.common.metrics.NopLongHistogram; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC; @@ -57,6 +60,7 @@ import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE; public class PopMetricsManager { + private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class); public static Supplier attributesBuilderSupplier; private static LongHistogram popBufferScanTimeConsume = new NopLongHistogram(); @@ -138,9 +142,13 @@ private static void calculatePopReviveLatency(BrokerController brokerController, ObservableLongMeasurement measurement) { PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices(); for (PopReviveService popReviveService : popReviveServices) { - measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder() - .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) - .build()); + try { + measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder() + .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) + .build()); + } catch (ConsumeQueueException e) { + log.error("Failed to get revive behind duration", e); + } } } @@ -148,9 +156,13 @@ private static void calculatePopReviveLag(BrokerController brokerController, ObservableLongMeasurement measurement) { PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices(); for (PopReviveService popReviveService : popReviveServices) { - measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder() - .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) - .build()); + try { + measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder() + .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) + .build()); + } catch (ConsumeQueueException e) { + log.error("Failed to get revive behind message count", e); + } } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java index 9896735dd1c..79bb0c771d6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.store.exception.ConsumeQueueException; /** * manage the offset of broadcast. @@ -72,7 +73,7 @@ public void updateOffset(String topic, String group, int queueId, long offset, S * @return -1 means no init offset, use the queueOffset in pullRequestHeader */ public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset, - boolean fromProxy) { + boolean fromProxy) throws ConsumeQueueException { BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId)); if (broadcastOffsetData == null) { @@ -84,29 +85,26 @@ public Long queryInitOffset(String topic, String groupId, int queueId, String cl } final AtomicLong offset = new AtomicLong(-1L); - broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdK, offsetStore) -> { - if (offsetStore == null) { - offsetStore = new BroadcastTimedOffsetStore(fromProxy); - } + BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId); + if (offsetStore == null) { + offsetStore = new BroadcastTimedOffsetStore(fromProxy); + broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore); + } - if (offsetStore.fromProxy && requestOffset < 0) { - // when from proxy and requestOffset is -1 - // means proxy need a init offset to pull message + if (offsetStore.fromProxy && requestOffset < 0) { + // when from proxy and requestOffset is -1 + // means proxy need a init offset to pull message + offset.set(getOffset(offsetStore, topic, groupId, queueId)); + } else { + if (offsetStore.fromProxy != fromProxy) { offset.set(getOffset(offsetStore, topic, groupId, queueId)); - return offsetStore; - } - - if (offsetStore.fromProxy == fromProxy) { - return offsetStore; } - - offset.set(getOffset(offsetStore, topic, groupId, queueId)); - return offsetStore; - }); + } return offset.get(); } - private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) { + private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) + throws ConsumeQueueException { long storeOffset = -1; if (offsetStore != null) { storeOffset = offsetStore.offsetStore.readOffset(queueId); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index dc1b1b53a32..043ef13f5a9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -20,6 +20,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.util.BitSet; +import java.nio.charset.StandardCharsets; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.KeyBuilder; @@ -30,7 +31,6 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -45,6 +45,7 @@ import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; @@ -134,7 +135,12 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re } long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset", e); + } if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d", requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset); @@ -166,7 +172,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re } private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, - final RemotingCommand response, final Channel channel, String brokerName) { + final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException { String[] extraInfo; String consumeGroup, topic; int qId, rqId; @@ -206,7 +212,12 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA invisibleTime = batchAck.getInvisibleTime(); long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (minOffset == -1 || maxOffset == -1) { POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck); return; @@ -254,7 +265,7 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(rqId); if (ackMsg instanceof BatchAckMsg) { msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 80f3f44facb..aa962513df3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -215,6 +215,7 @@ import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; @@ -1341,8 +1342,7 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); - final GetMaxOffsetRequestHeader requestHeader = - (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); + final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); @@ -1350,10 +1350,12 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, return rewriteResult; } - long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - - responseHeader.setOffset(offset); - + try { + long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + responseHeader.setOffset(offset); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -1484,7 +1486,8 @@ private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx, return response; } - private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) { + private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); HashMap runtimeInfo = this.prepareRuntimeInfo(); @@ -1686,8 +1689,8 @@ private RemotingCommand updateAndCreateSubscriptionGroupList(ChannelHandlerConte return response; } - - private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) { + private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) + throws ConsumeQueueException { String topic = topicConfig.getTopicName(); for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) { if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) { @@ -1761,8 +1764,7 @@ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetTopicStatsInfoRequestHeader requestHeader = - (GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); + final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); final String topic = requestHeader.getTopic(); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -1775,39 +1777,45 @@ private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, TopicStatsTable topicStatsTable = new TopicStatsTable(); int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums()); - for (int i = 0; i < maxQueueNums; i++) { - MessageQueue mq = new MessageQueue(); - mq.setTopic(topic); - mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); - mq.setQueueId(i); + try { + for (int i = 0; i < maxQueueNums; i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); - TopicOffset topicOffset = new TopicOffset(); - long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i); - if (min < 0) { - min = 0; - } + TopicOffset topicOffset = new TopicOffset(); + long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i); + if (min < 0) { + min = 0; + } - long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); - if (max < 0) { - max = 0; - } + long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + if (max < 0) { + max = 0; + } - long timestamp = 0; - if (max > 0) { - timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); - } + long timestamp = 0; + if (max > 0) { + timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); + } - topicOffset.setMinOffset(min); - topicOffset.setMaxOffset(max); - topicOffset.setLastUpdateTimestamp(timestamp); + topicOffset.setMinOffset(min); + topicOffset.setMaxOffset(max); + topicOffset.setLastUpdateTimestamp(timestamp); - topicStatsTable.getOffsetTable().put(mq, topicOffset); + topicStatsTable.getOffsetTable().put(mq, topicOffset); + } + + byte[] body = topicStatsTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); } - byte[] body = topicStatsTable.encode(); - response.setBody(body); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); return response; } @@ -1907,93 +1915,96 @@ private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx, private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetConsumeStatsRequestHeader requestHeader = - (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); - - ConsumeStats consumeStats = new ConsumeStats(); - - Set topics = new HashSet<>(); - if (UtilAll.isBlank(requestHeader.getTopic())) { - topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup()); - } else { - topics.add(requestHeader.getTopic()); - } + try { + final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); + ConsumeStats consumeStats = new ConsumeStats(); - for (String topic : topics) { - TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); - if (null == topicConfig) { - LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic); - continue; + Set topics = new HashSet<>(); + if (UtilAll.isBlank(requestHeader.getTopic())) { + topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup()); + } else { + topics.add(requestHeader.getTopic()); } - TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic); - - { - SubscriptionData findSubscriptionData = - this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); - - if (null == findSubscriptionData - && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) { - LOGGER.warn( - "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, " - + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup()); + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic); continue; } - } - for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { - MessageQueue mq = new MessageQueue(); - mq.setTopic(topic); - mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); - mq.setQueueId(i); + TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic); - OffsetWrapper offsetWrapper = new OffsetWrapper(); + { + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); - long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); - if (brokerOffset < 0) { - brokerOffset = 0; + if (null == findSubscriptionData + && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) { + LOGGER.warn( + "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, " + + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup()); + continue; + } } - long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset( - requestHeader.getConsumerGroup(), topic, i); + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + + OffsetWrapper offsetWrapper = new OffsetWrapper(); - // the consumerOffset cannot be zero for static topic because of the "double read check" strategy - // just remain the logic for dynamic topic - // maybe we should remove it in the future - if (mappingDetail == null) { - if (consumerOffset < 0) { - consumerOffset = 0; + long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + if (brokerOffset < 0) { + brokerOffset = 0; } - } - long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset( - requestHeader.getConsumerGroup(), topic, i); + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getConsumerGroup(), topic, i); + + // the consumerOffset cannot be zero for static topic because of the "double read check" strategy + // just remain the logic for dynamic topic + // maybe we should remove it in the future + if (mappingDetail == null) { + if (consumerOffset < 0) { + consumerOffset = 0; + } + } - offsetWrapper.setBrokerOffset(brokerOffset); - offsetWrapper.setConsumerOffset(consumerOffset); - offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset)); + long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset( + requestHeader.getConsumerGroup(), topic, i); - long timeOffset = consumerOffset - 1; - if (timeOffset >= 0) { - long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); - if (lastTimestamp > 0) { - offsetWrapper.setLastTimestamp(lastTimestamp); + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset)); + + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } } + + consumeStats.getOffsetTable().put(mq, offsetWrapper); } - consumeStats.getOffsetTable().put(mq, offsetWrapper); - } + double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic); - double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic); + consumeTps += consumeStats.getConsumeTps(); + consumeStats.setConsumeTps(consumeTps); + } - consumeTps += consumeStats.getConsumeTps(); - consumeStats.setConsumeTps(consumeTps); + byte[] body = consumeStats.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); } - - byte[] body = consumeStats.encode(); - response.setBody(body); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); return response; } @@ -2108,7 +2119,7 @@ public RemotingCommand resetOffset(ChannelHandlerContext ctx, requestHeader.getTimestamp(), requestHeader.isForce(), isC); } - private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) { + private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException { if (timestamp < 0) { return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); } else { @@ -2155,25 +2166,31 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId return response; } - if (queueId >= 0) { - if (null != offset && -1 != offset) { - long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); - long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); - if (min >= 0 && offset < min || offset > max + 1) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark( - String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max)); - return response; + try { + if (queueId >= 0) { + if (null != offset && -1 != offset) { + long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); + long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + if (min >= 0 && offset < min || offset > max + 1) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark( + String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max)); + return response; + } + } else { + offset = searchOffsetByTimestamp(topic, queueId, timestamp); } + queueOffsetMap.put(queueId, offset); } else { - offset = searchOffsetByTimestamp(topic, queueId, timestamp); - } - queueOffsetMap.put(queueId, offset); - } else { - for (int index = 0; index < topicConfig.getReadQueueNums(); index++) { - offset = searchOffsetByTimestamp(topic, index, timestamp); - queueOffsetMap.put(index, offset); + for (int index = 0; index < topicConfig.getReadQueueNums(); index++) { + offset = searchOffsetByTimestamp(topic, index, timestamp); + queueOffsetMap.put(index, offset); + } } + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); + return response; } if (queueOffsetMap.isEmpty()) { @@ -2280,8 +2297,7 @@ private RemotingCommand querySubscriptionByConsumer(ChannelHandlerContext ctx, private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - QueryConsumeTimeSpanRequestHeader requestHeader = - (QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); + QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); final String topic = requestHeader.getTopic(); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -2303,7 +2319,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i); timeSpan.setMinTimeStamp(minTime); - long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + long max; + try { + max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); timeSpan.setMaxTimeStamp(maxTime); @@ -2317,7 +2338,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, } timeSpan.setConsumeTimeStamp(consumeTime); - long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i); + long maxBrokerOffset; + try { + maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (consumerOffset < maxBrokerOffset) { long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset); timeSpan.setDelayTime(System.currentTimeMillis() - nextTime); @@ -2552,8 +2578,7 @@ private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx, private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - GetConsumeStatsInBrokerHeader requestHeader = - (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class); + GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class); boolean isOrder = requestHeader.isOrder(); ConcurrentMap subscriptionGroups = brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable(); @@ -2599,7 +2624,12 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); mq.setQueueId(i); OffsetWrapper offsetWrapper = new OffsetWrapper(); - long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + long brokerOffset; + try { + brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset", e); + } if (brokerOffset < 0) { brokerOffset = 0; } @@ -2643,7 +2673,7 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, return response; } - private HashMap prepareRuntimeInfo() { + private HashMap prepareRuntimeInfo() throws RemotingCommandException { HashMap runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo(); for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) { @@ -2652,7 +2682,11 @@ private HashMap prepareRuntimeInfo() { } } - this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo); + try { + this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } runtimeInfo.put("brokerActive", String.valueOf(this.brokerController.isSpecialServiceRunning())); runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)); runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION)); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index af3b8ae6f05..d29ff2a55b0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -21,6 +21,7 @@ import io.netty.channel.ChannelHandlerContext; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.nio.charset.StandardCharsets; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.PopAckConstants; @@ -30,7 +31,6 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -43,6 +43,7 @@ import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -120,7 +121,12 @@ public CompletableFuture processRequestAsync(final Channel chan return CompletableFuture.completedFuture(response); } long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max consume offset", e); + } if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { response.setCode(ResponseCode.NO_MESSAGE); return CompletableFuture.completedFuture(response); @@ -201,7 +207,7 @@ private CompletableFuture ackOrigin(final ChangeInvisibleTimeRequestHea } msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(rqId); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -244,7 +250,7 @@ private CompletableFuture appendCheckPointThenAckOrigin( ck.addDiff(0); ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo)); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java index c82725fe1e0..75c77b6d79f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader; import org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class NotificationProcessor implements NettyRequestProcessor { private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); @@ -169,13 +170,15 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, return response; } - private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) { + private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) + throws RemotingCommandException { boolean hasMsg; TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName); return hasMsgFromTopic(topicConfig, randomQ, requestHeader); } - private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) { + private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) + throws RemotingCommandException { boolean hasMsg; if (topicConfig != null) { for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { @@ -189,15 +192,19 @@ private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, Notificati return false; } - private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) { + private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) throws RemotingCommandException { if (Boolean.TRUE.equals(requestHeader.getOrder())) { if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) { return false; } } long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId); - long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset; - return restNum > 0; + try { + long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset; + return restNum > 0; + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed tp get max offset in queue", e); + } } private long getPopOffset(String topic, String cid, int queueId) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java index 2c0a1cd54a2..8473e3a2865 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java @@ -51,6 +51,7 @@ import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; @@ -229,13 +230,18 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult, PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel, - long popTime) { + long popTime) throws RemotingCommandException { String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2()) : requestHeader.getTopic(); GetMessageResult getMessageTmpResult; long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId); - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + } catch (ConsumeQueueException e) { + LOG.error("Failed to get max offset in queue. topic={}, queue-id={}", topic, queueId, e); + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) { return restNum; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 2d76c5a3caa..07bc0ac07b2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -24,6 +24,7 @@ import io.netty.channel.FileRegion; import io.opentelemetry.api.common.Attributes; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -62,7 +63,6 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.topic.TopicValidator; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -83,6 +83,7 @@ import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -167,13 +168,14 @@ public ConcurrentLinkedHashMap> getPol return popLongPollingService.getPollingMap(); } - public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) { + public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException { this.notifyLongPollingRequestIfNeed( topic, group, queueId, null, 0L, null, null); } public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId, - Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + Long tagsCode, long msgStoreTime, byte[] filterBitMap, + Map properties) throws ConsumeQueueException { long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId); long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId); long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); @@ -217,8 +219,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); - final PopMessageRequestHeader requestHeader = - (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true); + final PopMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true); StringBuilder startOffsetInfo = new StringBuilder(64); StringBuilder msgOffsetInfo = new StringBuilder(64); StringBuilder orderCountInfo = null; @@ -531,20 +532,37 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, String lockKey = topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId; boolean isOrder = requestHeader.isOrder(); - long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), - false, lockKey, false); + long offset; + try { + offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), + false, lockKey, false); + } catch (ConsumeQueueException e) { + CompletableFuture failure = new CompletableFuture<>(); + failure.completeExceptionally(e); + return failure; + } + CompletableFuture future = new CompletableFuture<>(); if (!queueLockManager.tryLock(lockKey)) { - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; - future.complete(restNum); + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + future.complete(restNum); + } catch (ConsumeQueueException e) { + future.completeExceptionally(e); + } return future; } future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) { - POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId); - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; - future.complete(restNum); + POP_LOGGER.warn("Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}", + topic, requestHeader.getConsumerGroup(), queueId); + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + future.complete(restNum); + } catch (ConsumeQueueException e) { + future.completeExceptionally(e); + } return future; } @@ -610,7 +628,11 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId, return CompletableFuture.completedFuture(result); }).thenApply(result -> { if (result == null) { - atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get()); + try { + atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get()); + } catch (ConsumeQueueException e) { + POP_LOGGER.error("Failed to get max offset in queue", e); + } return atomicRestNum.get(); } if (!result.getMessageMapedList().isEmpty()) { @@ -710,7 +732,7 @@ private boolean isPopShouldStop(String topic, String group, int queueId) { } private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey, - boolean checkResetOffset) { + boolean checkResetOffset) throws ConsumeQueueException { long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId); if (offset < 0) { @@ -732,7 +754,8 @@ private long getPopOffset(String topic, String group, int queueId, int initMode, } } - private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) { + private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) + throws ConsumeQueueException { long offset; if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); @@ -761,7 +784,7 @@ public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 4b141d29102..f27934efdfd 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -19,6 +19,7 @@ import com.alibaba.fastjson.JSON; import io.opentelemetry.api.common.Attributes; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -54,6 +55,7 @@ import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -260,10 +262,14 @@ public PullResult getMessage(String group, String topic, int queueId, long offse getMessageResult.getMaxOffset(), foundList); } else { - long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); - if (maxQueueOffset > offset) { - POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}", - topic, group, offset, maxQueueOffset); + try { + long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + if (maxQueueOffset > offset) { + POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}", + topic, group, offset, maxQueueOffset); + } + } catch (ConsumeQueueException e) { + POP_LOGGER.error("Failed to get max offset in queue", e); } return null; } @@ -364,7 +370,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { firstRt = point.getReviveTime(); } } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -388,7 +394,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { } } } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -594,7 +600,7 @@ private void rePutCK(PopCheckPoint oldCK, Pair pair) { brokerController.getMessageStore().putMessage(ckMsg); } - public long getReviveBehindMillis() { + public long getReviveBehindMillis() throws ConsumeQueueException { if (currentReviveMessageTimestamp <= 0) { return 0; } @@ -605,7 +611,7 @@ public long getReviveBehindMillis() { return 0; } - public long getReviveBehindMessages() { + public long getReviveBehindMessages() throws ConsumeQueueException { if (currentReviveMessageTimestamp <= 0) { return 0; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index 6dd8b300478..2ad2c9e93e4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -73,6 +73,7 @@ import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.stats.BrokerStatsManager; import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; @@ -298,7 +299,8 @@ public boolean rejectRequest() { return false; } - private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, boolean brokerAllowFlowCtrSuspend) + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, + boolean brokerAllowFlowCtrSuspend) throws RemotingCommandException { final long beginTimeMills = this.brokerController.getMessageStore().now(); RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); @@ -489,7 +491,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re final MessageStore messageStore = brokerController.getMessageStore(); if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) { - DefaultMessageStore defaultMessageStore = (DefaultMessageStore)this.brokerController.getMessageStore(); + DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore(); boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup()); if (cgNeedColdDataFlowCtr) { boolean isMsgLogicCold = defaultMessageStore.getCommitLog() @@ -526,7 +528,11 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET); getMessageResult.setNextBeginOffset(resetOffset); getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId)); - getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId)); + try { + getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId)); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed tp get max offset in queue", e); + } getMessageResult.setSuggestPullingFromSlave(false); } else { long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel); @@ -589,12 +595,13 @@ public boolean hasConsumeMessageHook() { /** * Composes the header of the response message to be sent back to the client - * @param requestHeader - the header of the request message - * @param getMessageResult - the result of the GetMessage request - * @param topicSysFlag - the system flag of the topic + * + * @param requestHeader - the header of the request message + * @param getMessageResult - the result of the GetMessage request + * @param topicSysFlag - the system flag of the topic * @param subscriptionGroupConfig - configuration of the subscription group - * @param response - the response message to be sent back to the client - * @param clientAddress - the address of the client + * @param response - the response message to be sent back to the client + * @param clientAddress - the address of the client */ protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult, int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response, @@ -855,7 +862,7 @@ protected void updateBroadcastPulledOffset(String topic, String group, int queue * When pull request is not broadcast or not return -1 */ protected long queryBroadcastPullInitOffset(String topic, String group, int queueId, - PullMessageRequestHeader requestHeader, Channel channel) { + PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException { if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) { return -1L; @@ -877,8 +884,12 @@ protected long queryBroadcastPullInitOffset(String topic, String group, int queu clientId = clientChannelInfo.getClientId(); } - return this.brokerController.getBroadcastOffsetManager() - .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast); + try { + return this.brokerController.getBroadcastOffsetManager() + .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to query initial offset", e); + } } return -1L; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java index e13b36df910..70184e8a620 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java @@ -53,6 +53,7 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; @@ -103,7 +104,7 @@ public static int delayLevel2QueueId(final int delayLevel) { return delayLevel - 1; } - public void buildRunningStats(HashMap stats) { + public void buildRunningStats(HashMap stats) throws ConsumeQueueException { for (Map.Entry next : this.offsetTable.entrySet()) { int queueId = delayLevel2QueueId(next.getKey()); long delayOffset = next.getValue(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java index 865e7b608ea..7e16d329e1b 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.ResponseCode; @@ -125,14 +126,14 @@ public void testCheckProducerTransactionStateException() throws Exception { } @Test - public void testResetOffsetNoTopicConfig() { + public void testResetOffsetNoTopicConfig() throws RemotingCommandException { when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null); RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); } @Test - public void testResetOffsetNoConsumerGroupInfo() { + public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException { TopicConfig topicConfig = mock(TopicConfig.class); when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); when(topicConfig.getWriteQueueNums()).thenReturn(1); @@ -142,7 +143,7 @@ public void testResetOffsetNoConsumerGroupInfo() { } @Test - public void testResetOffset() { + public void testResetOffset() throws RemotingCommandException { TopicConfig topicConfig = mock(TopicConfig.class); when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); when(topicConfig.getWriteQueueNums()).thenReturn(1); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java index 9dc00f9d6b1..ad5af92646e 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.broker.client.ConsumerManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -51,7 +52,7 @@ public class BroadcastOffsetManagerTest { private BroadcastOffsetManager broadcastOffsetManager; @Before - public void before() { + public void before() throws ConsumeQueueException { brokerConfig.setEnableBroadcastOffsetStore(true); brokerConfig.setBroadcastOffsetExpireSecond(1); brokerConfig.setBroadcastOffsetExpireMaxSecond(5); @@ -84,7 +85,7 @@ public void before() { } @Test - public void testBroadcastOffsetSwitch() { + public void testBroadcastOffsetSwitch() throws ConsumeQueueException { // client1 connect to broker onlineClientIdSet.add("client1"); long offset = broadcastOffsetManager.queryInitOffset("group", "topic", 0, "client1", 0, false); @@ -160,4 +161,4 @@ public void testBroadcastOffsetExpire() { return broadcastOffsetManager.offsetStoreMap.isEmpty(); }); } -} \ No newline at end of file +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java index 93689efa586..1fdf454d5e0 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java @@ -22,6 +22,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.processor.PopMessageProcessor; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; @@ -50,7 +51,7 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest { private final BrokerController brokerController = mock(BrokerController.class); @Before - public void before() { + public void before() throws ConsumeQueueException { notified = new AtomicBoolean(false); brokerConfig.setEnableNotifyAfterPopOrderLockRelease(true); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); @@ -175,4 +176,4 @@ public void testRecover() { await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get); assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty()); } -} \ No newline at end of file +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java index c0afb46c330..757b01b63ff 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java @@ -47,6 +47,7 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -93,7 +94,7 @@ public class AckMessageProcessorTest { private static final long MAX_OFFSET_IN_QUEUE = 999; @Before - public void init() throws IllegalAccessException, NoSuchFieldException { + public void init() throws IllegalAccessException, NoSuchFieldException, ConsumeQueueException { clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); brokerController.setMessageStore(messageStore); Field field = BrokerController.class.getDeclaredField("broker2Client"); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java index 8a2ce8a2ba4..fdb0690e5dc 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.junit.Assert; import org.junit.Before; @@ -182,7 +183,7 @@ public void testGetInitOffset_retryTopic() throws RemotingCommandException { } @Test - public void testGetInitOffset_normalTopic() throws RemotingCommandException { + public void testGetInitOffset_normalTopic() throws RemotingCommandException, ConsumeQueueException { long maxOffset = 999L; when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset); diff --git a/client/BUILD.bazel b/client/BUILD.bazel index 9b6fbc298c2..b93f3d90996 100644 --- a/client/BUILD.bazel +++ b/client/BUILD.bazel @@ -33,6 +33,7 @@ java_library( "@maven//:commons_collections_commons_collections", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", + "@maven//:com_google_guava_guava", ], ) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 0e5571eb130..716d081ef46 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -1199,9 +1199,9 @@ private PopResult processPopResponse(final String brokerName, final RemotingComm messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) { // process LMQ String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); long offset = Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]); // LMQ topic has only 1 queue, which queue id is 0 queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID); @@ -1264,9 +1264,9 @@ private static Map> buildQueueOffsetSortedMap(String topic, L && StringUtils.isNotEmpty(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) { // process LMQ String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); // LMQ topic has only 1 queue, which queue id is 0 key = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID); sortMap.putIfAbsent(key, new ArrayList<>(4)); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index e311e0c9b85..81dc5883fb8 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -636,8 +636,8 @@ public void testPopMultiLmqMessage_async() throws Exception { final int invisibleTime = 10 * 1000; final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1"; final String lmqTopic2 = MixAll.LMQ_PREFIX + "lmq2"; - final String multiDispatch = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, lmqTopic, lmqTopic2); - final String multiOffset = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, "0", "0"); + final String multiDispatch = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, lmqTopic, lmqTopic2); + final String multiOffset = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, "0", "0"); doAnswer((Answer) mock -> { InvokeCallback callback = mock.getArgument(3); RemotingCommand request = mock.getArgument(1); diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index efb115509ac..39933038bac 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.IOTinyUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -99,8 +100,8 @@ public class MixAll { public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; public static final String REPLY_MESSAGE_FLAG = "reply"; public static final String LMQ_PREFIX = "%LMQ%"; - public static final long LMQ_QUEUE_ID = 0; - public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ","; + public static final int LMQ_QUEUE_ID = 0; + public static final String LMQ_DISPATCH_SEPARATOR = ","; public static final String REQ_T = "ReqT"; public static final String ROCKETMQ_ZONE_ENV = "ROCKETMQ_ZONE"; public static final String ROCKETMQ_ZONE_PROPERTY = "rocketmq.zone"; @@ -524,4 +525,10 @@ public static boolean isSysConsumerGroupForNoColdReadLimit(String consumerGroup) } return false; } + + public static boolean topicAllowsLMQ(String topic) { + return !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) + && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index 95dc8b9800b..96195d53090 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -24,7 +24,7 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public abstract class ServiceThread implements Runnable { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + protected static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private static final long JOIN_TIME = 90 * 1000; diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java index 95d5119cfc6..a4ba35bd5ae 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java @@ -16,8 +16,8 @@ */ package org.apache.rocketmq.common.config; +import com.google.common.base.Strings; import java.io.File; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; @@ -110,12 +110,26 @@ public static DBOptions createConfigDBOptions() { } public static String getDBLogDir() { - String rootPath = System.getProperty("user.home"); - if (StringUtils.isEmpty(rootPath)) { - return ""; + String[] rootPaths = new String[] { + System.getProperty("user.home"), + System.getProperty("java.io.tmpdir"), + File.separator + "data" + }; + for (String rootPath : rootPaths) { + // Refer bazel test encyclopedia: https://bazel.build/reference/test-encyclopedia + // Not all directories is available + if (Strings.isNullOrEmpty(rootPath)) { + continue; + } + File rootPathFile = new File(rootPath); + if (!rootPathFile.exists() || !rootPathFile.canWrite()) { + continue; + } + String logDirectory = rootPath + File.separator + "logs" + File.separator + "rocketmqlogs"; + // Create directories recursively. + UtilAll.ensureDirOK(logDirectory); + return logDirectory; } - rootPath = rootPath + File.separator + "logs"; - UtilAll.ensureDirOK(rootPath); - return rootPath + File.separator + "rocketmqlogs" + File.separator; + throw new RuntimeException("Failed to get log directory"); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java index 147f23f1234..e4f02e2c9b4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java @@ -16,8 +16,11 @@ */ package org.apache.rocketmq.common.message; +import com.google.common.base.Strings; import java.nio.ByteBuffer; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.utils.MessageUtils; @@ -41,7 +44,7 @@ public void setEncodedBuff(ByteBuffer encodedBuff) { } public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { - if (null == tags || tags.length() == 0) { return 0; } + if (Strings.isNullOrEmpty(tags)) { return 0; } return tags.hashCode(); } @@ -102,4 +105,9 @@ public boolean isEncodeCompleted() { public void setEncodeCompleted(boolean encodeCompleted) { this.encodeCompleted = encodeCompleted; } + + public boolean needDispatchLMQ() { + return StringUtils.isNoneBlank(getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) + && MixAll.topicAllowsLMQ(getTopic()); + } } diff --git a/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java new file mode 100644 index 00000000000..1fcc967d677 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.config; + +import org.junit.Test; + +public class ConfigHelperTest { + + @Test + public void testGetDBLogDir() { + // Should not raise exception. + ConfigHelper.getDBLogDir(); + } + +} diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java index 5fee9480287..da6ad920f30 100644 --- a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java @@ -49,7 +49,7 @@ public static void main(String[] args) throws MQClientException, InterruptedExce Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); msg.setKeys("Key" + i); msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */, - String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); + String.join(MixAll.LMQ_DISPATCH_SEPARATOR, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); SendResult sendResult = producer.send(msg); System.out.printf("%s%n", sendResult); } catch (Exception e) { diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java index 6c7327f258e..0e35acc01f9 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java @@ -49,7 +49,7 @@ public void testEncode() throws IOException { random.nextBytes(data); write(file, data); FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength); - Assert.assertEquals(0, fileRegion.transfered()); + Assert.assertEquals(0, fileRegion.transferred()); Assert.assertEquals(dataLength, fileRegion.count()); Assert.assertTrue(channel.writeOutbound(fileRegion)); ByteBuf out = (ByteBuf) channel.readOutbound(); @@ -77,4 +77,4 @@ private static void write(File file, byte[] data) throws IOException { } } } -} \ No newline at end of file +} diff --git a/store/BUILD.bazel b/store/BUILD.bazel index 8364a239c9a..98f90a577cf 100644 --- a/store/BUILD.bazel +++ b/store/BUILD.bazel @@ -42,6 +42,8 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:commons_validator_commons_validator", ], ) @@ -63,6 +65,7 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:org_junit_jupiter_junit_jupiter_api", ], ) diff --git a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java index 4d53f3b7bcd..c3534d06113 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java +++ b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java @@ -25,4 +25,5 @@ public enum AppendMessageStatus { MESSAGE_SIZE_EXCEEDED, PROPERTIES_SIZE_EXCEEDED, UNKNOWN_ERROR, + ROCKSDB_ERROR, } diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 972e71aadd8..153215c98ad 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store; +import com.google.common.base.Strings; import java.net.Inet6Address; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -35,7 +36,6 @@ import java.util.stream.Collectors; import com.sun.jna.NativeLong; import com.sun.jna.Pointer; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.SystemClock; @@ -58,10 +58,11 @@ import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; -import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -104,7 +105,6 @@ public class CommitLog implements Swappable { protected int commitLogSize; private final boolean enabledAppendPropCRC; - protected final MultiDispatch multiDispatch; public CommitLog(final DefaultMessageStore messageStore) { String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog(); @@ -139,8 +139,6 @@ protected PutMessageThreadLocal initialValue() { this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); this.enabledAppendPropCRC = messageStore.getMessageStoreConfig().isEnabledAppendPropCRC(); - - this.multiDispatch = new MultiDispatch(defaultMessageStore); } public void setFullStorePaths(Set fullStorePaths) { @@ -530,7 +528,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, } String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS); - if (tags != null && tags.length() > 0) { + if (!Strings.isNullOrEmpty(tags)) { tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags); } @@ -652,7 +650,7 @@ public long getConfirmOffset() { } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) { return this.confirmOffset; } else { - return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset(); + return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset(); } } @@ -770,8 +768,11 @@ else if (size == 0) { } } - // only for rocksdb mode - this.getMessageStore().finishCommitLogDispatch(); + try { + this.getMessageStore().getQueueStore().flush(); + } catch (StoreException e) { + log.error("Failed to flush ConsumeQueueStore", e); + } processOffset += mappedFileOffset; if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { @@ -988,7 +989,7 @@ public CompletableFuture asyncPutMessage(final MessageExtBroke msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer()); PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey); - putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + putMessageLock.lock(); //spin or ReentrantLock, depending on store config try { long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); this.beginTimeInLock = beginLockTimestamp; @@ -1850,7 +1851,16 @@ public AppendMessageResult handlePropertiesForLmqMsg(ByteBuffer preEncodeBuffer, return null; } - multiDispatch.wrapMultiDispatch(msgInner); + try { + LmqDispatch.wrapLmqDispatch(defaultMessageStore, msgInner); + } catch (ConsumeQueueException e) { + if (e.getCause() instanceof RocksDBException) { + log.error("Failed to wrap multi-dispatch", e); + return new AppendMessageResult(AppendMessageStatus.ROCKSDB_ERROR); + } + log.error("Failed to wrap multi-dispatch", e); + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); @@ -1904,7 +1914,7 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
    ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff(); - final boolean isMultiDispatchMsg = CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner); + boolean isMultiDispatchMsg = messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ(); if (isMultiDispatchMsg) { AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner); if (appendMessageResult != null) { @@ -2000,7 +2010,12 @@ public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer msgInner.setEncodedBuff(null); if (isMultiDispatchMsg) { - CommitLog.this.multiDispatch.updateMultiQueueOffset(msgInner); + try { + LmqDispatch.updateLmqOffsets(defaultMessageStore, msgInner); + } catch (ConsumeQueueException e) { + // Increase in-memory max offset of the queue should not fail. + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } } return new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgIdSupplier, @@ -2245,11 +2260,6 @@ public FlushManager getFlushManager() { return flushManager; } - public static boolean isMultiDispatchMsg(MessageStoreConfig messageStoreConfig, MessageExtBrokerInner msg) { - return StringUtils.isNotBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && - MultiDispatchUtils.isNeedHandleMultiDispatch(messageStoreConfig, msg.getTopic()); - } - private boolean isCloseReadAhead() { return !MixAll.isWindows() && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index eb8af4ab190..b6b9cff538d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -727,8 +727,8 @@ private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { Map prop = request.getPropertiesMap(); String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR); if (queues.length != queueOffsets.length) { log.error("[bug] queues.length!=queueOffsets.length ", request.getTopic()); return; diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 8b46c7f5ce4..6b8ea0ee8ad 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -93,6 +93,7 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.dledger.DLedgerCommitLog; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.DefaultHAService; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; @@ -170,7 +171,7 @@ public class DefaultMessageStore implements MessageStore { private RocksDBMessageStore rocksDBMessageStore; - private RandomAccessFile lockFile; + private final RandomAccessFile lockFile; private FileLock lock; @@ -190,7 +191,7 @@ public class DefaultMessageStore implements MessageStore { private volatile long brokerInitMaxOffset = -1L; - private List putMessageHookList = new ArrayList<>(); + private final List putMessageHookList = new ArrayList<>(); private SendMessageBackHook sendMessageBackHook; @@ -203,20 +204,21 @@ public class DefaultMessageStore implements MessageStore { private final ConcurrentLinkedQueue batchDispatchRequestQueue = new ConcurrentLinkedQueue<>(); - private int dispatchRequestOrderlyQueueSize = 16; + private final int dispatchRequestOrderlyQueueSize = 16; private final DispatchRequestOrderlyQueue dispatchRequestOrderlyQueue = new DispatchRequestOrderlyQueue(dispatchRequestOrderlyQueueSize); private long stateMachineVersion = 0L; // this is a unmodifiableMap - private ConcurrentMap topicConfigTable; + private final ConcurrentMap topicConfigTable; private final ScheduledExecutorService scheduledCleanQueueExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, - final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, + final ConcurrentMap topicConfigTable) throws IOException { this.messageArrivingListener = messageArrivingListener; this.brokerConfig = brokerConfig; this.messageStoreConfig = messageStoreConfig; @@ -438,7 +440,7 @@ private void doRecheckReputOffsetFromCq() throws InterruptedException { return; } - /** + /* * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog; * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go; * 3. Calculate the reput offset according to the consume queue; @@ -458,7 +460,7 @@ private void doRecheckReputOffsetFromCq() throws InterruptedException { } if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) { maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset(); - /** + /* * This happens in following conditions: * 1. If someone removes all the consumequeue files or the disk get damaged. * 2. Launch a new broker, and copy the commitlog from other brokers. @@ -987,12 +989,12 @@ public CompletableFuture getMessageAsync(String group, String } @Override - public long getMaxOffsetInQueue(String topic, int queueId) { + public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException { return getMaxOffsetInQueue(topic, queueId, true); } @Override - public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException { if (committed) { ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId); if (logic != null) { @@ -1378,7 +1380,6 @@ public long now() { * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, * consume queue will be created with old offset, then later message with new offset table can not be * dispatched to consume queue. - * @throws RocksDBException only in rocksdb mode */ @Override public int deleteTopics(final Set deleteTopics) { @@ -1748,10 +1749,11 @@ public boolean checkInDiskByCommitOffset(long offsetPy) { /** * The ratio val is estimated by the experiment and experience * so that the result is not high accurate for different business + * * @return */ public boolean checkInColdAreaByCommitOffset(long offsetPy, long maxOffsetPy) { - long memory = (long)(StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0)); + long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0)); return (maxOffsetPy - offsetPy) > memory; } @@ -1929,11 +1931,6 @@ public MessageStoreConfig getMessageStoreConfig() { return messageStoreConfig; } - @Override - public void finishCommitLogDispatch() { - // ignore - } - @Override public TransientStorePool getTransientStorePool() { return transientStorePool; @@ -2713,15 +2710,15 @@ public long getJoinTime() { } } - class BatchDispatchRequest { + static class BatchDispatchRequest { - private ByteBuffer byteBuffer; + private final ByteBuffer byteBuffer; - private int position; + private final int position; - private int size; + private final int size; - private long id; + private final long id; public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long id) { this.byteBuffer = byteBuffer; @@ -2731,7 +2728,7 @@ public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long } } - class DispatchRequestOrderlyQueue { + static class DispatchRequestOrderlyQueue { DispatchRequest[][] buffer; @@ -2907,8 +2904,6 @@ public void doReput() { } finally { result.release(); } - - finishCommitLogDispatch(); } } @@ -2922,8 +2917,8 @@ private void notifyMessageArrive4MultiQueue(DispatchRequest dispatchRequest) { if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { return; } - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR); if (queues.length != queueOffsets.length) { return; } @@ -2932,7 +2927,7 @@ private void notifyMessageArrive4MultiQueue(DispatchRequest dispatchRequest) { long queueOffset = Long.parseLong(queueOffsets[i]); int queueId = dispatchRequest.getQueueId(); if (DefaultMessageStore.this.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; + queueId = MixAll.LMQ_QUEUE_ID; } DefaultMessageStore.this.messageArrivingListener.arriving( queueName, queueId, queueOffset + 1, dispatchRequest.getTagsCode(), @@ -2972,13 +2967,13 @@ class MainBatchDispatchRequestService extends ServiceThread { public MainBatchDispatchRequestService() { batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( - DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), - DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), - 1000 * 60, - TimeUnit.MICROSECONDS, - new LinkedBlockingQueue<>(4096), - new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"), - new ThreadPoolExecutor.AbortPolicy()); + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + 1000 * 60, + TimeUnit.MICROSECONDS, + new LinkedBlockingQueue<>(4096), + new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"), + new ThreadPoolExecutor.AbortPolicy()); } private void pollBatchDispatchRequest() { @@ -3188,9 +3183,6 @@ public void doReput() { result.release(); } } - - // only for rocksdb mode - finishCommitLogDispatch(); } /** diff --git a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java index 79d006bafc3..654760b88c8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java +++ b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java @@ -17,6 +17,9 @@ package org.apache.rocketmq.store; import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; public class DispatchRequest { private final String topic; @@ -228,6 +231,18 @@ public void setOffsetId(String offsetId) { this.offsetId = offsetId; } + public boolean containsLMQ() { + if (!MixAll.topicAllowsLMQ(topic)) { + return false; + } + if (null == propertiesMap || propertiesMap.isEmpty()) { + return false; + } + String lmqNames = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String lmqOffsets = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + return !StringUtils.isBlank(lmqNames) && !StringUtils.isBlank(lmqOffsets); + } + @Override public String toString() { return "DispatchRequest{" + diff --git a/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java new file mode 100644 index 00000000000..2805f510140 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.exception.ConsumeQueueException; + +public class LmqDispatch { + private static final short VALUE_OF_EACH_INCREMENT = 1; + + public static void wrapLmqDispatch(MessageStore messageStore, final MessageExtBrokerInner msg) + throws ConsumeQueueException { + String lmqNames = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + Long[] queueOffsets = new Long[queueNames.length]; + if (messageStore.getMessageStoreConfig().isEnableLmq()) { + for (int i = 0; i < queueNames.length; i++) { + if (MixAll.isLmq(queueNames[i])) { + queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(queueNames[i], MixAll.LMQ_QUEUE_ID); + } + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.LMQ_DISPATCH_SEPARATOR)); + msg.removeWaitStorePropertyString(); + } + + public static void updateLmqOffsets(MessageStore messageStore, final MessageExtBrokerInner msgInner) + throws ConsumeQueueException { + String lmqNames = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + for (String queueName : queueNames) { + if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { + messageStore.getQueueStore().increaseLmqOffset(queueName, MixAll.LMQ_QUEUE_ID, VALUE_OF_EACH_INCREMENT); + } + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java index 5c74918d9e6..7531c96d119 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java @@ -175,11 +175,11 @@ public PutMessageResult encodeWithoutProperties(MessageExtBrokerInner msgInner) public PutMessageResult encode(MessageExtBrokerInner msgInner) { this.byteBuf.clear(); - if (CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner)) { + if (messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ()) { return encodeWithoutProperties(msgInner); } - /** + /* * Serialize message */ final byte[] propertiesData = diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 814c6d1bfef..5c3984e5b2c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import javax.annotation.Nonnull; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.SystemClock; @@ -31,6 +32,7 @@ import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; @@ -181,7 +183,7 @@ CompletableFuture getMessageAsync(final String group, final St * @param queueId Queue ID. * @return Maximum offset at present. */ - long getMaxOffsetInQueue(final String topic, final int queueId); + long getMaxOffsetInQueue(final String topic, final int queueId) throws ConsumeQueueException; /** * Get maximum offset of the topic queue. @@ -191,7 +193,7 @@ CompletableFuture getMessageAsync(final String group, final St * @param committed return the max offset in ConsumeQueue if true, or the max offset in CommitLog if false * @return Maximum offset at present. */ - long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed); + long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed) throws ConsumeQueueException; /** * Get the minimum offset of the topic queue. @@ -626,14 +628,6 @@ CompletableFuture queryMessageAsync(final String topic, fina void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, boolean isRecover, boolean isFileEnd) throws RocksDBException; - /** - * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) - * It will be triggered in two cases: - * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput - * @see CommitLog#recoverAbnormally - */ - void finishCommitLogDispatch(); - /** * Get the message store config * @@ -724,6 +718,7 @@ void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, Ma * * @return the queue store */ + @Nonnull ConsumeQueueStoreInterface getQueueStore(); /** diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java deleted file mode 100644 index 5bc587a8e03..00000000000 --- a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.store; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; - -/** - * MultiDispatch for lmq, not-thread-safe - */ -public class MultiDispatch { - private final StringBuilder keyBuilder = new StringBuilder(); - private final DefaultMessageStore messageStore; - private static final short VALUE_OF_EACH_INCREMENT = 1; - - public MultiDispatch(DefaultMessageStore messageStore) { - this.messageStore = messageStore; - } - - public String queueKey(String queueName, MessageExtBrokerInner msgInner) { - keyBuilder.delete(0, keyBuilder.length()); - keyBuilder.append(queueName); - keyBuilder.append('-'); - int queueId = msgInner.getQueueId(); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; - } - keyBuilder.append(queueId); - return keyBuilder.toString(); - } - - public void wrapMultiDispatch(final MessageExtBrokerInner msg) { - - String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - Long[] queueOffsets = new Long[queues.length]; - if (messageStore.getMessageStoreConfig().isEnableLmq()) { - for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (MixAll.isLmq(key)) { - queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(key); - } - } - } - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, - StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); - msg.removeWaitStorePropertyString(); - } - - public void updateMultiQueueOffset(final MessageExtBrokerInner msgInner) { - String multiDispatchQueue = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - for (String queue : queues) { - String key = queueKey(queue, msgInner); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { - messageStore.getQueueStore().increaseLmqOffset(key, VALUE_OF_EACH_INCREMENT); - } - } - } -} \ No newline at end of file diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 21f8d45c9d9..0a7119cab1d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -79,15 +79,6 @@ public void recoverTopicQueueTable() { this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); } - @Override - public void finishCommitLogDispatch() { - try { - putMessagePositionInfo(null); - } catch (RocksDBException e) { - ERROR_LOG.info("try to finish commitlog dispatch error.", e); - } - } - @Override public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { return findConsumeQueue(topic, queueId); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 5195868e0f1..8effe35bab6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -428,8 +428,6 @@ public class MessageStoreConfig { private boolean rocksdbCQDoubleWriteEnable = false; - private int batchWriteKvCqSize = 16; - /** * If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type. * The following values are valid: @@ -447,14 +445,6 @@ public class MessageStoreConfig { */ private String bottomMostCompressionTypeForConsumeQueueStore = "zstd"; - public int getBatchWriteKvCqSize() { - return batchWriteKvCqSize; - } - - public void setBatchWriteKvCqSize(int batchWriteKvCqSize) { - this.batchWriteKvCqSize = batchWriteKvCqSize; - } - public boolean isRocksdbCQDoubleWriteEnable() { return rocksdbCQDoubleWriteEnable; } diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java new file mode 100644 index 00000000000..880e6347eb4 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.exception; + +public class ConsumeQueueException extends StoreException { + public ConsumeQueueException() { + } + + public ConsumeQueueException(String message) { + super(message); + } + + public ConsumeQueueException(String message, Throwable cause) { + super(message, cause); + } + + public ConsumeQueueException(Throwable cause) { + super(cause); + } + + public ConsumeQueueException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java new file mode 100644 index 00000000000..8c99e8a05bb --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.exception; + +public class StoreException extends Exception { + public StoreException() { + } + + public StoreException(String message) { + super(message); + } + + public StoreException(String message, Throwable cause) { + super(message, cause); + } + + public StoreException(Throwable cause) { + super(cause); + } + + public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 2401257c306..0f57a17d463 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -46,6 +46,7 @@ import org.apache.rocketmq.store.StoreStatsService; import org.apache.rocketmq.store.TransientStorePool; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; @@ -63,7 +64,7 @@ import io.opentelemetry.sdk.metrics.ViewBuilder; public abstract class AbstractPluginMessageStore implements MessageStore { - protected MessageStore next = null; + protected MessageStore next; protected MessageStorePluginContext context; public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) { @@ -139,12 +140,12 @@ public CompletableFuture getMessageAsync(String group, String } @Override - public long getMaxOffsetInQueue(String topic, int queueId) { + public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException { return next.getMaxOffsetInQueue(topic, queueId); } @Override - public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException { return next.getMaxOffsetInQueue(topic, queueId, committed); } @@ -647,11 +648,6 @@ public void initMetrics(Meter meter, Supplier attributesBuild next.initMetrics(meter, attributesBuilderSupplier); } - @Override - public void finishCommitLogDispatch() { - next.finishCommitLogDispatch(); - } - @Override public void recoverTopicQueueTable() { next.recoverTopicQueueTable(); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java index d76b0557737..dfce665d8fa 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.rocksdb.RocksDBException; public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { @@ -47,7 +48,7 @@ public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, Di } @Override - public Long getMaxOffset(String topic, int queueId) { + public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException { return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); } @@ -58,7 +59,7 @@ public void setTopicQueueTable(ConcurrentMap topicQueueTable) { } @Override - public ConcurrentMap getTopicQueueTable() { + public ConcurrentMap getTopicQueueTable() { return this.queueOffsetOperator.getTopicQueueTable(); } @@ -75,13 +76,13 @@ public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { } @Override - public void increaseLmqOffset(String queueKey, short messageNum) { - queueOffsetOperator.increaseLmqOffset(queueKey, messageNum); + public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException { + queueOffsetOperator.increaseLmqOffset(topic, queueId, delta); } @Override - public long getLmqQueueOffset(String queueKey) { - return queueOffsetOperator.getLmqOffset(queueKey); + public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException { + return queueOffsetOperator.getLmqOffset(topic, queueId, (t, q) -> 0L); } @Override @@ -105,9 +106,9 @@ public long getStoreTime(CqUnit cqUnit) { try { final long phyOffset = cqUnit.getPos(); final int size = cqUnit.getSize(); - long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); - return storeTime; + return this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); } catch (Exception e) { + log.error("Failed to getStoreTime", e); } } return -1; diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java index cbe9b4f5acd..5f9c2f90be3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java @@ -47,6 +47,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.StoreException; import static java.lang.String.format; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; @@ -134,6 +135,12 @@ public boolean recoverConcurrently() { @Override public boolean shutdown() { + try { + flush(); + } catch (StoreException e) { + log.error("Failed to flush all consume queues", e); + return false; + } return true; } @@ -326,6 +333,15 @@ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { return fileQueueLifeCycle.flush(flushLeastPages); } + @Override + public void flush() throws StoreException { + for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { + for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { + flush(cqEntry.getValue(), 0); + } + } + } + @Override public void destroy(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java index e68880a828c..72a481bd57b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java @@ -22,6 +22,8 @@ import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.rocksdb.RocksDBException; public interface ConsumeQueueStoreInterface { @@ -79,10 +81,17 @@ public interface ConsumeQueueStoreInterface { boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); /** - * clean expired data from minPhyOffset - * @param minPhyOffset + * Flush all nested consume queues to disk + * + * @throws StoreException if there is an error during flush + */ + void flush() throws StoreException; + + /** + * clean expired data from minCommitLogOffset + * @param minCommitLogOffset Minimum commit log offset */ - void cleanExpired(long minPhyOffset); + void cleanExpired(long minCommitLogOffset); /** * Check files. @@ -92,10 +101,10 @@ public interface ConsumeQueueStoreInterface { /** * Delete expired files ending at min commit log position. * @param consumeQueue - * @param minCommitLogPos min commit log position + * @param minCommitLogOffset min commit log position * @return deleted file numbers. */ - int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); + int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogOffset); /** * Is the first file available? @@ -185,17 +194,19 @@ public interface ConsumeQueueStoreInterface { /** * Increase lmq offset - * @param queueKey - * @param messageNum + * @param topic Topic/Queue name + * @param queueId Queue ID + * @param delta amount to increase */ - void increaseLmqOffset(String queueKey, short messageNum); + void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException; /** * get lmq queue offset - * @param queueKey + * @param topic + * @param queueId * @return */ - long getLmqQueueOffset(String queueKey); + long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException; /** * recover topicQueue table by minPhyOffset @@ -232,11 +243,13 @@ public interface ConsumeQueueStoreInterface { /** * get maxOffset of specific topic-queueId in topicQueue table - * @param topic - * @param queueId + * + * @param topic Topic name + * @param queueId Queue identifier * @return the max offset in QueueOffsetOperator + * @throws ConsumeQueueException if there is an error while retrieving max consume queue offset */ - Long getMaxOffset(String topic, int queueId); + Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException; /** * get max physic offset in consumeQueue diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java new file mode 100644 index 00000000000..a93ec0c50e1 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue; + +import java.nio.charset.StandardCharsets; +import javax.annotation.Nonnull; +import org.apache.rocketmq.store.DispatchRequest; + +/** + * Use Record when Java 16 is available + */ +public class DispatchEntry { + public byte[] topic; + public int queueId; + public long queueOffset; + public long commitLogOffset; + public int messageSize; + public long tagCode; + public long storeTimestamp; + + public static DispatchEntry from(@Nonnull DispatchRequest request) { + DispatchEntry entry = new DispatchEntry(); + entry.topic = request.getTopic().getBytes(StandardCharsets.UTF_8); + entry.queueId = request.getQueueId(); + entry.queueOffset = request.getConsumeQueueOffset(); + entry.commitLogOffset = request.getCommitLogOffset(); + entry.messageSize = request.getMsgSize(); + entry.tagCode = request.getTagsCode(); + entry.storeTimestamp = request.getStoreTimestamp(); + return entry; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java new file mode 100644 index 00000000000..8d3a8cd3a49 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.store.exception.ConsumeQueueException; + +public interface OffsetInitializer { + long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException; +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java new file mode 100644 index 00000000000..4b889e1e448 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.rocksdb.RocksDBException; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + +public class OffsetInitializerRocksDBImpl implements OffsetInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(OffsetInitializerRocksDBImpl.class); + + private final RocksDBConsumeQueueStore consumeQueueStore; + + public OffsetInitializerRocksDBImpl(RocksDBConsumeQueueStore consumeQueueStore) { + this.consumeQueueStore = consumeQueueStore; + } + + @Override + public long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException { + try { + long offset = consumeQueueStore.getMaxOffsetInQueue(topic, queueId); + LOGGER.info("Look up RocksDB for max-offset of LMQ[{}:{}]: {}", topic, queueId, offset); + return offset; + } catch (RocksDBException e) { + throw new ConsumeQueueException(e); + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java index 5b4bf994e0e..7d388171618 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.queue; +import com.google.common.base.Preconditions; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -26,6 +27,7 @@ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.exception.ConsumeQueueException; /** * QueueOffsetOperator is a component for operating offsets for queues. @@ -35,7 +37,11 @@ public class QueueOffsetOperator { private ConcurrentMap topicQueueTable = new ConcurrentHashMap<>(1024); private ConcurrentMap batchTopicQueueTable = new ConcurrentHashMap<>(1024); - private ConcurrentMap lmqTopicQueueTable = new ConcurrentHashMap<>(1024); + + /** + * {TOPIC}-{QUEUE_ID} --> NEXT Consume Queue Offset + */ + private ConcurrentMap lmqTopicQueueTable = new ConcurrentHashMap<>(1024); public long getQueueOffset(String topicQueueKey) { return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); @@ -63,17 +69,28 @@ public void increaseBatchQueueOffset(String topicQueueKey, short messageNum) { this.batchTopicQueueTable.put(topicQueueKey, batchQueueOffset + messageNum); } - public long getLmqOffset(String topicQueueKey) { - return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); - } - - public Long getLmqTopicQueueNextOffset(String topicQueueKey) { - return this.lmqTopicQueueTable.get(topicQueueKey); + public long getLmqOffset(String topic, int queueId, OffsetInitializer callback) throws ConsumeQueueException { + Preconditions.checkNotNull(callback, "ConsumeQueueOffsetCallback cannot be null"); + String topicQueue = topic + "-" + queueId; + if (!lmqTopicQueueTable.containsKey(topicQueue)) { + // Load from RocksDB on cache miss. + Long prev = lmqTopicQueueTable.putIfAbsent(topicQueue, callback.maxConsumeQueueOffset(topic, queueId)); + if (null != prev) { + log.error("[BUG] Data racing, lmqTopicQueueTable should NOT contain key={}", topicQueue); + } + } + return lmqTopicQueueTable.get(topicQueue); } - public void increaseLmqOffset(String queueKey, short messageNum) { - Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, queueKey, k -> 0L); - this.lmqTopicQueueTable.put(queueKey, lmqOffset + messageNum); + public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException { + String topicQueue = topic + "-" + queueId; + if (!this.lmqTopicQueueTable.containsKey(topicQueue)) { + throw new ConsumeQueueException(String.format("Max offset of Queue[name=%s, id=%d] should have existed", topic, queueId)); + } + long prev = lmqTopicQueueTable.get(topicQueue); + this.lmqTopicQueueTable.compute(topicQueue, (k, offset) -> offset + delta); + long current = lmqTopicQueueTable.get(topicQueue); + log.debug("Max offset of LMQ[{}:{}] increased: {} --> {}", topic, queueId, prev, current); } public long currentQueueOffset(String topicQueueKey) { @@ -112,4 +129,4 @@ public ConcurrentMap getTopicQueueTable() { public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { this.batchTopicQueueTable = batchTopicQueueTable; } -} \ No newline at end of file +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java index 6fa66282e9d..889131d1cc8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java @@ -16,7 +16,10 @@ */ package org.apache.rocketmq.store.queue; +import io.netty.util.internal.PlatformDependent; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -24,31 +27,33 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; -import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.queue.offset.OffsetEntry; +import org.apache.rocketmq.store.queue.offset.OffsetEntryType; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; -import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1; public class RocksDBConsumeQueueOffsetTable { private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); - private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); + private static final byte[] MAX_BYTES = "max".getBytes(StandardCharsets.UTF_8); + private static final byte[] MIN_BYTES = "min".getBytes(StandardCharsets.UTF_8); /** * Rocksdb ConsumeQueue's Offset unit. Format: @@ -72,10 +77,9 @@ public class RocksDBConsumeQueueOffsetTable { * * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes */ - private static final int OFFSET_PHY_OFFSET = 0; - private static final int OFFSET_CQ_OFFSET = 8; + static final int OFFSET_PHY_OFFSET = 0; + static final int OFFSET_CQ_OFFSET = 8; /** - * * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ @@ -86,16 +90,18 @@ public class RocksDBConsumeQueueOffsetTable { /** * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. + * * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of * RocksDBConsumeQueueOffsetTable to find it. */ private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; - private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(StandardCharsets.UTF_8); private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; private final ByteBuffer maxPhyOffsetBB; + static { buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); @@ -111,69 +117,147 @@ public class RocksDBConsumeQueueOffsetTable { /** * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them * from heap to avoid accessing rocksdb. + * * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset */ - private final Map topicQueueMinOffset; - private final Map topicQueueMaxCqOffset; + private final ConcurrentMap topicQueueMinOffset; + private final ConcurrentMap topicQueueMaxCqOffset; public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; this.rocksDBStorage = rocksDBStorage; this.messageStore = messageStore; - this.topicQueueMinOffset = new ConcurrentHashMap(1024); - this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); + this.topicQueueMinOffset = new ConcurrentHashMap<>(1024); + this.topicQueueMaxCqOffset = new ConcurrentHashMap<>(1024); this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); } public void load() { this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); + loadMaxConsumeQueueOffsets(); } - public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, - final byte[] topicBytes, final DispatchRequest request, - final Map> tempTopicQueueMaxOffsetMap) { - buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); - ByteBuffer topicQueueId = offsetBBPair.getObject1(); - ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); - Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); - if (old == null) { - tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); - } else { - long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); - long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); - if (maxOffset >= oldMaxOffset) { - ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); + private void loadMaxConsumeQueueOffsets() { + Function predicate = entry -> entry.type == OffsetEntryType.MAXIMUM; + Consumer fn = entry -> { + topicQueueMaxCqOffset.putIfAbsent(entry.topic + "-" + entry.queueId, entry.offset); + ROCKSDB_LOG.info("Max {}:{} --> {}|{}", entry.topic, entry.queueId, entry.offset, entry.commitLogOffset); + }; + try { + forEach(predicate, fn); + } catch (RocksDBException e) { + log.error("Failed to maximum consume queue offset", e); + } + } + + public void forEach(Function predicate, Consumer fn) throws RocksDBException { + try (RocksIterator iterator = this.rocksDBStorage.seekOffsetCF()) { + if (null == iterator) { + return; + } + + int keyBufferCapacity = 256; + iterator.seekToFirst(); + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity); + ByteBuffer valueBuffer = ByteBuffer.allocateDirect(16); + while (iterator.isValid()) { + // parse key buffer according to key layout + keyBuffer.clear(); // clear position and limit before reuse + int total = iterator.key(keyBuffer); + if (total > keyBufferCapacity) { + keyBufferCapacity = total; + PlatformDependent.freeDirectBuffer(keyBuffer); + keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity); + continue; + } + + if (keyBuffer.remaining() <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES) { + iterator.next(); + ROCKSDB_LOG.warn("Malformed Key/Value pair"); + continue; + } + + int topicLength = keyBuffer.getInt(); + byte ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + + byte[] topicBytes = new byte[topicLength]; + keyBuffer.get(topicBytes); + ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + String topic = new String(topicBytes, StandardCharsets.UTF_8); + + byte[] minMax = new byte[3]; + keyBuffer.get(minMax); + OffsetEntryType entryType; + if (Arrays.equals(minMax, MAX_BYTES)) { + entryType = OffsetEntryType.MAXIMUM; + } else { + entryType = OffsetEntryType.MINIMUM; + } + ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + + assert keyBuffer.remaining() == Integer.BYTES; + int queueId = keyBuffer.getInt(); + + // Read and parse value buffer according to value layout + valueBuffer.clear(); // clear position and limit before reuse + total = iterator.value(valueBuffer); + if (total != Long.BYTES + Long.BYTES) { + // Skip system checkpoint topic as its value is only 8 bytes + iterator.next(); + continue; + } + long commitLogOffset = valueBuffer.getLong(); + long consumeOffset = valueBuffer.getLong(); + + OffsetEntry entry = new OffsetEntry(); + entry.topic = topic; + entry.queueId = queueId; + entry.type = entryType; + entry.offset = consumeOffset; + entry.commitLogOffset = commitLogOffset; + if (predicate.apply(entry)) { + fn.accept(entry); + } + iterator.next(); } + // clean up direct buffers + PlatformDependent.freeDirectBuffer(keyBuffer); + PlatformDependent.freeDirectBuffer(valueBuffer); } } - public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, + public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { - for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); } appendMaxPhyOffset(writeBatch, maxPhyOffset); } - public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { - for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { - DispatchRequest request = entry.getValue().getObject2(); - putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); + public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + DispatchEntry dispatchEntry = entry.getValue().getObject2(); + String topic = new String(dispatchEntry.topic, StandardCharsets.UTF_8); + putHeapMaxCqOffset(topic, dispatchEntry.queueId, dispatchEntry.queueOffset); } } /** * When topic is deleted, we clean up its offset info in rocksdb. + * * @param topic * @param queueId * @throws RocksDBException */ public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; @@ -214,15 +298,14 @@ public long getMaxPhyOffset() throws RocksDBException { /** * Traverse the offset table to find dirty topic + * * @param existTopicSet * @return */ public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { Map> topicQueueIdToBeDeletedMap = new HashMap<>(); - RocksIterator iterator = null; - try { - iterator = rocksDBStorage.seekOffsetCF(); + try (RocksIterator iterator = rocksDBStorage.seekOffsetCF()) { if (iterator == null) { return topicQueueIdToBeDeletedMap; } @@ -236,17 +319,22 @@ public Map> iterateOffsetTable2FindDirty(final Set ByteBuffer keyBB = ByteBuffer.wrap(key); int topicLen = keyBB.getInt(0); byte[] topicBytes = new byte[topicLen]; - /** + /* * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 */ keyBB.position(4 + 1); keyBB.get(topicBytes); - String topic = new String(topicBytes, CHARSET_UTF8); + String topic = new String(topicBytes, StandardCharsets.UTF_8); if (TopicValidator.isSystemTopic(topic)) { continue; } - /** + // LMQ topic offsets should NOT be removed + if (MixAll.isLmq(topic)) { + continue; + } + + /* * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" * = 4 + 1 + topicLen + 1 + 3 + 1 */ @@ -270,10 +358,6 @@ public Map> iterateOffsetTable2FindDirty(final Set } } catch (Exception e) { ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); - } finally { - if (iterator != null) { - iterator.close(); - } } return topicQueueIdToBeDeletedMap; } @@ -285,9 +369,13 @@ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; String topicQueueId = buildTopicQueueId(topic, queueId); - this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); + long offset = maxCqOffset != null ? maxCqOffset : -1L; + Long prev = this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, offset); + if (null == prev) { + ROCKSDB_LOG.info("Max offset of {} is initialized to {} according to RocksDB", topicQueueId, offset); + } if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { - ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); + ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, offset); } } @@ -296,34 +384,50 @@ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { /** * truncate dirty offset in rocksdb + * * @param offsetToTruncate * @throws RocksDBException */ public void truncateDirty(long offsetToTruncate) throws RocksDBException { correctMaxPyhOffset(offsetToTruncate); - ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); - if (allTopicConfigMap == null) { - return; - } - for (TopicConfig topicConfig : allTopicConfigMap.values()) { - for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { - truncateDirtyOffset(topicConfig.getTopicName(), i); + Function predicate = entry -> { + if (entry.type == OffsetEntryType.MINIMUM) { + return false; } - } + // Normal entry offset MUST have the following inequality + // entry commit-log offset + message-size-in-bytes <= offsetToTruncate; + // otherwise, the consume queue contains dirty records to truncate; + // + // If the broker node is configured to use async-flush, it's possible consume queues contain + // pointers to message records that is not flushed and lost during restart. + return entry.commitLogOffset >= offsetToTruncate; + }; + + Consumer fn = entry -> { + try { + truncateDirtyOffset(entry.topic, entry.queueId); + } catch (RocksDBException e) { + log.error("Failed to truncate maximum offset of consume queue[topic={}, queue-id={}]", + entry.topic, entry.queueId, e); + } + }; + + forEach(predicate, fn); } - private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { + private Pair isMinOffsetOk(final String topic, final int queueId, + final long minPhyOffset) throws RocksDBException { PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); if (phyAndCQOffset != null) { final long phyOffset = phyAndCQOffset.getPhyOffset(); final long cqOffset = phyAndCQOffset.getCqOffset(); - return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); + return (phyOffset >= minPhyOffset) ? new Pair<>(true, cqOffset) : new Pair<>(false, cqOffset); } ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); if (byteBuffer == null) { - return new Pair(false, 0L); + return new Pair<>(false, 0L); } final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); @@ -334,9 +438,9 @@ private Pair isMinOffsetOk(final String topic, final int queueId, if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); } - return new Pair(true, cqOffset); + return new Pair<>(true, cqOffset); } - return new Pair(false, cqOffset); + return new Pair<>(false, cqOffset); } private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { @@ -361,8 +465,7 @@ private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException { if (!this.rocksDBStorage.hold()) { return; } - try { - WriteBatch writeBatch = new WriteBatch(); + try (WriteBatch writeBatch = new WriteBatch()) { long oldMaxPhyOffset = getMaxPhyOffset(); if (oldMaxPhyOffset <= maxPhyOffset) { return; @@ -416,10 +519,10 @@ private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws Ro } private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); - byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); + byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); return (value != null) ? ByteBuffer.wrap(value) : null; } @@ -427,17 +530,19 @@ private String buildTopicQueueId(final String topic, final int queueId) { return topic + "-" + queueId; } - private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { + private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, + final long minCQOffset) { String topicQueueId = buildTopicQueueId(topic, queueId); PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); } - private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { + private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxOffset) { String topicQueueId = buildTopicQueueId(topic, queueId); - Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); - if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { - ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); + Long prev = this.topicQueueMaxCqOffset.put(topicQueueId, maxOffset); + if (prev != null && prev > maxOffset) { + ERROR_LOG.error("Max offset of consume-queue[topic={}, queue-id={}] regressed. prev-max={}, current-max={}", + topic, queueId, prev, maxOffset); } } @@ -463,9 +568,8 @@ private void updateCqOffset(final String topic, final int queueId, final long ph if (!this.rocksDBStorage.hold()) { return; } - WriteBatch writeBatch = new WriteBatch(); - try { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + try (WriteBatch writeBatch = new WriteBatch()) { + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); @@ -481,7 +585,6 @@ private void updateCqOffset(final String topic, final int queueId, final long ph ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); throw e; } finally { - writeBatch.close(); this.rocksDBStorage.release(); if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", @@ -504,10 +607,8 @@ private boolean correctMaxCqOffset(final String topic, final int queueId, final throw new RocksDBException("correctMaxCqOffset error"); } - long high = maxCQOffset; - long low = minCQOffset; - PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, - low, maxPhyOffsetInCQ, false); + PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset, + minCQOffset, maxPhyOffsetInCQ, false); long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); @@ -541,10 +642,8 @@ private boolean correctMinCqOffset(final String topic, final int queueId, return true; } - long high = maxCQOffset; - long low = minCQOffset; - PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, - minPhyOffset, true); + PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset, + minCQOffset, minPhyOffset, true); long targetCQOffset = phyAndCQOffset.getCqOffset(); long targetPhyOffset = phyAndCQOffset.getPhyOffset(); @@ -568,28 +667,29 @@ public static Pair getOffsetByteBufferPair() { return new Pair<>(offsetKey, offsetValue); } - private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, - final byte[] topicBytes, final DispatchRequest request) { + static void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, + final DispatchEntry entry) { final ByteBuffer offsetKey = offsetBBPair.getObject1(); - buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); + buildOffsetKeyByteBuffer(offsetKey, entry.topic, entry.queueId, true); final ByteBuffer offsetValue = offsetBBPair.getObject2(); - buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); + buildOffsetValueByteBuffer(offsetValue, entry.commitLogOffset, entry.queueOffset); } - private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { + private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); return byteBuffer; } - private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { + private static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, + final int queueId, final boolean max) { byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); } - private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, - final boolean max) { + private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, + final int queueId, final boolean max) { byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); if (max) { byteBuffer.put(MAX_BYTES); @@ -600,18 +700,20 @@ private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byteBuffer.flip(); } - private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + private static void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, + final long cqOffset) { byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); } - private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { + private static ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); return byteBuffer; } - private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + private static void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, + final long cqOffset) { byteBuffer.putLong(phyOffset).putLong(cqOffset); byteBuffer.flip(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 17b845d8176..67a00157431 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -18,6 +18,7 @@ import java.io.File; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -28,21 +29,27 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + +import javax.annotation.Nonnull; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.FlushOptions; import org.rocksdb.RocksDBException; import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; @@ -51,11 +58,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - public static final byte CTRL_0 = '\u0000'; - public static final byte CTRL_1 = '\u0001'; - public static final byte CTRL_2 = '\u0002'; + private static final int DEFAULT_BYTE_BUFFER_CAPACITY = 16; - private final int batchSize; public static final int MAX_KEY_LEN = 300; private final ScheduledExecutorService scheduledExecutorService; @@ -70,13 +74,16 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; - private final WriteBatch writeBatch; - private final List bufferDRList; private final List> cqBBPairList; private final List> offsetBBPairList; - private final Map> tempTopicQueueMaxOffsetMap; + private final Map> tempTopicQueueMaxOffsetMap; private volatile boolean isCQError = false; + private int consumeQueueByteBufferCacheIndex; + private int offsetBufferCacheIndex; + + private final OffsetInitializer offsetInitializer; + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); @@ -85,12 +92,10 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); - this.writeBatch = new WriteBatch(); - this.batchSize = messageStoreConfig.getBatchWriteKvCqSize(); - this.bufferDRList = new ArrayList<>(batchSize); - this.cqBBPairList = new ArrayList<>(batchSize); - this.offsetBBPairList = new ArrayList<>(batchSize); - for (int i = 0; i < batchSize; i++) { + this.offsetInitializer = new OffsetInitializerRocksDBImpl(this); + this.cqBBPairList = new ArrayList<>(16); + this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY); + for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) { this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); } @@ -100,6 +105,22 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); } + private Pair getCQByteBufferPair() { + int idx = consumeQueueByteBufferCacheIndex++; + if (idx >= cqBBPairList.size()) { + this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); + } + return cqBBPairList.get(idx); + } + + private Pair getOffsetByteBufferPair() { + int idx = offsetBufferCacheIndex++; + if (idx >= offsetBBPairList.size()) { + this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); + } + return offsetBBPairList.get(idx); + } + @Override public void start() { log.info("RocksDB ConsumeQueueStore start!"); @@ -164,19 +185,19 @@ private boolean shutdownInner() { @Override public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { - if (request == null || this.bufferDRList.size() >= batchSize) { - putMessagePosition(); - } - - if (request != null) { - this.bufferDRList.add(request); + if (null == request) { + return; } + // We are taking advantage of Atomic Flush, this operation is purely memory-based. + // batch and cache in Java heap does not make sense, instead, we should put the metadata into RocksDB immediately + // to optimized overall end-to-end latency. + putMessagePosition(request); } - public void putMessagePosition() throws RocksDBException { + public void putMessagePosition(DispatchRequest request) throws RocksDBException { final int maxRetries = 30; for (int i = 0; i < maxRetries; i++) { - if (putMessagePosition0()) { + if (putMessagePosition0(request)) { if (this.isCQError) { this.messageStore.getRunningFlags().clearLogicsQueueError(); this.isCQError = false; @@ -198,81 +219,113 @@ public void putMessagePosition() throws RocksDBException { throw new RocksDBException("put CQ Failed"); } - private boolean putMessagePosition0() { + private boolean putMessagePosition0(DispatchRequest request) { if (!this.rocksDBStorage.hold()) { return false; } - final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; - try { - final List bufferDRList = this.bufferDRList; - final int size = bufferDRList.size(); - if (size == 0) { - return true; - } - final List> cqBBPairList = this.cqBBPairList; - final List> offsetBBPairList = this.offsetBBPairList; - final WriteBatch writeBatch = this.writeBatch; - + try (WriteBatch writeBatch = new WriteBatch()) { long maxPhyOffset = 0; - for (int i = size - 1; i >= 0; i--) { - final DispatchRequest request = bufferDRList.get(i); - final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); - - this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); - this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), - topicBytes, request, tempTopicQueueMaxOffsetMap); - - final int msgSize = request.getMsgSize(); - final long phyOffset = request.getCommitLogOffset(); - if (phyOffset + msgSize >= maxPhyOffset) { - maxPhyOffset = phyOffset + msgSize; - } + DispatchEntry entry = DispatchEntry.from(request); + dispatch(entry, writeBatch); + dispatchLMQ(request, writeBatch); + + final int msgSize = request.getMsgSize(); + final long phyOffset = request.getCommitLogOffset(); + if (phyOffset + msgSize >= maxPhyOffset) { + maxPhyOffset = phyOffset + msgSize; } this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); - // clear writeBatch in batchPut this.rocksDBStorage.batchPut(writeBatch); - this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); - - long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); + long storeTimeStamp = request.getStoreTimestamp(); if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); } this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); - - notifyMessageArriveAndClear(); + notifyMessageArrival(request); return true; } catch (Exception e) { - ERROR_LOG.error("putMessagePosition0 Failed.", e); + ERROR_LOG.error("putMessagePosition0 failed.", e); return false; } finally { tempTopicQueueMaxOffsetMap.clear(); + consumeQueueByteBufferCacheIndex = 0; + offsetBufferCacheIndex = 0; this.rocksDBStorage.release(); } } - private void notifyMessageArriveAndClear() { - final List bufferDRList = this.bufferDRList; - try { - for (DispatchRequest dp : bufferDRList) { - this.messageStore.notifyMessageArriveIfNecessary(dp); + private void dispatch(@Nonnull DispatchEntry entry, @Nonnull final WriteBatch writeBatch) throws RocksDBException { + this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(getCQByteBufferPair(), entry, writeBatch); + updateTempTopicQueueMaxOffset(getOffsetByteBufferPair(), entry); + } + + private void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, + final DispatchEntry entry) { + RocksDBConsumeQueueOffsetTable.buildOffsetKeyAndValueByteBuffer(offsetBBPair, entry); + ByteBuffer topicQueueId = offsetBBPair.getObject1(); + ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); + Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); + if (old == null) { + tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair<>(maxOffsetBB, entry)); + } else { + long oldMaxOffset = old.getObject1().getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET); + long maxOffset = maxOffsetBB.getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET); + if (maxOffset >= oldMaxOffset) { + ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); } + } + } + + private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteBatch writeBatch) + throws RocksDBException { + if (!messageStoreConfig.isEnableLmq() || !request.containsLMQ()) { + return; + } + Map map = request.getPropertiesMap(); + String lmqNames = map.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String lmqOffsets = map.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + String[] queues = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = lmqOffsets.split(MixAll.LMQ_DISPATCH_SEPARATOR); + if (queues.length != queueOffsets.length) { + ERROR_LOG.error("[bug] queues.length!=queueOffsets.length ", request.getTopic()); + return; + } + for (int i = 0; i < queues.length; i++) { + String queueName = queues[i]; + DispatchEntry entry = DispatchEntry.from(request); + long queueOffset = Long.parseLong(queueOffsets[i]); + int queueId = request.getQueueId(); + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { + queueId = MixAll.LMQ_QUEUE_ID; + } + entry.queueId = queueId; + entry.queueOffset = queueOffset; + entry.topic = queueName.getBytes(StandardCharsets.UTF_8); + log.debug("Dispatch LMQ[{}:{}]:{} --> {}", queueName, queueId, queueOffset, entry.commitLogOffset); + dispatch(entry, writeBatch); + } + } + + private void notifyMessageArrival(DispatchRequest request) { + try { + this.messageStore.notifyMessageArriveIfNecessary(request); } catch (Exception e) { ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); - } finally { - bufferDRList.clear(); } } public Statistics getStatistics() { return rocksDBStorage.getStatistics(); } + @Override - public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + public List rangeQuery(final String topic, final int queueId, final long startIndex, + final int num) throws RocksDBException { return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); } @@ -284,6 +337,7 @@ public ByteBuffer get(final String topic, final int queueId, final long cqOffset /** * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them * when we use them, we call it lazy correction. + * * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) */ @@ -310,8 +364,7 @@ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException return; } - WriteBatch writeBatch = new WriteBatch(); - try { + try (WriteBatch writeBatch = new WriteBatch()) { this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); @@ -320,7 +373,6 @@ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); throw e; } finally { - writeBatch.close(); this.rocksDBStorage.release(); } } @@ -330,10 +382,22 @@ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { try { this.rocksDBStorage.flushWAL(); } catch (Exception e) { + log.error("Failed to flush WAL", e); } return true; } + @Override + public void flush() throws StoreException { + try (FlushOptions flushOptions = new FlushOptions()) { + flushOptions.setWaitForFlush(true); + flushOptions.setAllowWriteStall(true); + this.rocksDBStorage.flush(flushOptions); + } catch (RocksDBException e) { + throw new StoreException(e); + } + } + @Override public void checkSelf() { // ignored @@ -350,8 +414,9 @@ public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitL * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). - * @param offsetToTruncate - * @throws RocksDBException + * + * @param offsetToTruncate CommitLog offset to truncate to + * @throws RocksDBException If there is any error. */ @Override public void truncateDirty(long offsetToTruncate) throws RocksDBException { @@ -369,7 +434,8 @@ public void cleanExpired(final long minPhyOffset) { } @Override - public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, + BoundaryType boundaryType) throws RocksDBException { final long minPhysicOffset = this.messageStore.getMinPhyOffset(); long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); @@ -380,6 +446,15 @@ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, Bo minPhysicOffset, boundaryType); } + /** + * This method actually returns NEXT slot index to use, starting from 0. For example, if the queue is empty, + * it returns 0, pointing to the first slot of the 0-based queue; + * + * @param topic Topic name + * @param queueId Queue ID + * @return Index of the next slot to push into + * @throws RocksDBException if RocksDB fails to fulfill the request. + */ @Override public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); @@ -444,4 +519,17 @@ public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { public long getTotalSize() { return 0; } + + @Override + public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException { + return queueOffsetOperator.getLmqOffset(topic, queueId, offsetInitializer); + } + + @Override + public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException { + if (MixAll.isLmq(topic)) { + return getLmqQueueOffset(topic, queueId); + } + return super.getMaxOffset(topic, queueId); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java index 194bd4cca5f..deee0295706 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.queue; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -26,17 +27,15 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.WriteBatch; -import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_0; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_2; /** * We use RocksDBConsumeQueueTable to store cqUnit. @@ -105,30 +104,30 @@ public void load() { this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); } - public void buildAndPutCQByteBuffer(final Pair cqBBPair, - final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { + public void buildAndPutCQByteBuffer(final Pair cqBBPair, final DispatchEntry request, + final WriteBatch writeBatch) throws RocksDBException { final ByteBuffer cqKey = cqBBPair.getObject1(); - buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); + buildCQKeyByteBuffer(cqKey, request.topic, request.queueId, request.queueOffset); final ByteBuffer cqValue = cqBBPair.getObject2(); - buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); + buildCQValueByteBuffer(cqValue, request.commitLogOffset, request.messageSize, request.tagCode, request.storeTimestamp); writeBatch.put(this.defaultCFH, cqKey, cqValue); } public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); return (value != null) ? ByteBuffer.wrap(value) : null; } public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); - final List defaultCFHList = new ArrayList(num); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); + final List defaultCFHList = new ArrayList<>(num); final ByteBuffer[] resultList = new ByteBuffer[num]; - final List kvIndexList = new ArrayList(num); - final List kvKeyList = new ArrayList(num); + final List kvIndexList = new ArrayList<>(num); + final List kvKeyList = new ArrayList<>(num); for (int i = 0; i < num; i++) { final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); kvIndexList.add(i); @@ -153,7 +152,7 @@ public List rangeQuery(final String topic, final int queueId, final } final int resultSize = resultList.length; - List bbValueList = new ArrayList(resultSize); + List bbValueList = new ArrayList<>(resultSize); for (int i = 0; i < resultSize; i++) { ByteBuffer byteBuffer = resultList[i]; if (byteBuffer == null) { @@ -171,7 +170,7 @@ public List rangeQuery(final String topic, final int queueId, final * @throws RocksDBException */ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java new file mode 100644 index 00000000000..bac3aa430b4 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue.offset; + +public class OffsetEntry { + /** + * Topic identifier. For now, it's topic name directly. In the future, we should use fixed length topic identifier. + */ + public String topic; + + /** + * Queue ID + */ + public int queueId; + + /** + * Flag if the entry is for maximum or minimum + */ + public OffsetEntryType type; + + /** + * Maximum or minimum consume-queue offset. + */ + public long offset; + + /** + * Maximum or minimum commit-log offset. + */ + public long commitLogOffset; +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java new file mode 100644 index 00000000000..78df4e15fa5 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue.offset; + +public enum OffsetEntryType { + MAXIMUM, + MINIMUM +} diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 1d09ca86ecb..eee38e0a8f4 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -54,6 +54,7 @@ import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -374,7 +375,7 @@ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { } @Test - public void testPutMessage_whenMessagePropertyIsTooLong() { + public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException { String topicName = "messagePropertyIsTooLongTest"; MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); @@ -539,7 +540,7 @@ public void testGroupCommit() throws Exception { } @Test - public void testMaxOffset() throws InterruptedException { + public void testMaxOffset() throws InterruptedException, ConsumeQueueException { int firstBatchMessages = 3; int queueId = 0; messageBody = storeMessage.getBytes(); diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java deleted file mode 100644 index eae5eaa07a2..00000000000 --- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.store; - -import java.io.File; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; - -import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.apache.rocketmq.store.queue.MultiDispatchUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.rocksdb.RocksDBException; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class MultiDispatchTest { - - private MultiDispatch multiDispatch; - - private DefaultMessageStore messageStore; - - @Before - public void init() throws Exception { - MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); - messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); - messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); - messageStoreConfig.setMaxHashSlotNum(100); - messageStoreConfig.setMaxIndexNum(100 * 10); - messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1"); - messageStoreConfig.setStorePathCommitLog( - System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1" + File.separator + "commitlog"); - - messageStoreConfig.setEnableLmq(true); - messageStoreConfig.setEnableMultiDispatch(true); - BrokerConfig brokerConfig = new BrokerConfig(); - //too much reference - messageStore = new DefaultMessageStore(messageStoreConfig, null, null, brokerConfig, new ConcurrentHashMap<>()); - multiDispatch = new MultiDispatch(messageStore); - } - - @After - public void destroy() { - UtilAll.deleteFile(new File(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1")); - } - - @Test - public void lmqQueueKey() { - MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); - when(messageExtBrokerInner.getQueueId()).thenReturn(2); - String ret = MultiDispatchUtils.lmqQueueKey("%LMQ%lmq123"); - assertEquals(ret, "%LMQ%lmq123-0"); - } - - @Test - public void wrapMultiDispatch() throws RocksDBException { - MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); - multiDispatch.wrapMultiDispatch(messageExtBrokerInner); - assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); - } - - private MessageExtBrokerInner buildMessageMultiQueue() { - MessageExtBrokerInner msg = new MessageExtBrokerInner(); - msg.setTopic("test"); - msg.setTags("TAG1"); - msg.setKeys("Hello"); - msg.setBody("aaa".getBytes(Charset.forName("UTF-8"))); - msg.setKeys(String.valueOf(System.currentTimeMillis())); - msg.setQueueId(0); - msg.setSysFlag(0); - msg.setBornTimestamp(System.currentTimeMillis()); - msg.setStoreHost(new InetSocketAddress("127.0.0.1", 54270)); - msg.setBornHost(new InetSocketAddress("127.0.0.1", 10911)); - for (int i = 0; i < 1; i++) { - msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, "%LMQ%123,%LMQ%456"); - } - msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); - - return msg; - } -} \ No newline at end of file diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java index 2af07197a50..f934f803641 100644 --- a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java @@ -56,6 +56,7 @@ import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -434,7 +435,7 @@ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { } @Test - public void testPutMessage_whenMessagePropertyIsTooLong() { + public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException { if (notExecuted()) { return; } @@ -603,7 +604,7 @@ public void testGroupCommit() { } @Test - public void testMaxOffset() { + public void testMaxOffset() throws ConsumeQueueException { if (notExecuted()) { return; } diff --git a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java index ad521440e9e..bfed96e8cdf 100644 --- a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java @@ -36,6 +36,7 @@ import org.apache.rocketmq.remoting.protocol.filter.FilterAPI; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.store.DefaultMessageFilter; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -80,7 +81,7 @@ public void tearDown() { shutdown(); } - private Pair getLag(List mqs) { + private Pair getLag(List mqs) throws ConsumeQueueException { long lag = 0; long pullLag = 0; for (BrokerController controller : brokerControllerList) { @@ -120,7 +121,7 @@ public void waitForFullyDispatched() { } @Test - public void testCalculateLag() { + public void testCalculateLag() throws ConsumeQueueException { int msgSize = 10; List mqs = producer.getMessageQueue(); MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java index ee06700b8b0..982909c5ee5 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java @@ -34,6 +34,7 @@ import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.tieredstore.MessageStoreConfig; @@ -259,6 +260,10 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, } ); } + } catch (ConsumeQueueException e) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; } finally { flatFile.getFileLock().unlock(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java index 8b5a9e63c0c..4d083284834 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.metrics.NopLongHistogram; import org.apache.rocketmq.common.metrics.NopObservableLongGauge; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher; @@ -177,26 +178,30 @@ public static void init(Meter meter, Supplier attributesBuild .ofLongs() .buildWithCallback(measurement -> { for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) { + try { - MessageQueue mq = flatFile.getMessageQueue(); - long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); - long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { - continue; - } + MessageQueue mq = flatFile.getMessageQueue(); + long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); + long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { + continue; + } + + Attributes commitLogAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) + .build(); - Attributes commitLogAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) - .build(); - - Attributes consumeQueueAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) - .build(); - measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes); + Attributes consumeQueueAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) + .build(); + measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes); + } catch (ConsumeQueueException e) { + // TODO: handle exception here + } } }); @@ -206,31 +211,35 @@ public static void init(Meter meter, Supplier attributesBuild .ofLongs() .buildWithCallback(measurement -> { for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) { + try { + MessageQueue mq = flatFile.getMessageQueue(); - MessageQueue mq = flatFile.getMessageQueue(); - long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); - long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { - continue; - } + long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); + long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { + continue; + } - Attributes commitLogAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) - .build(); - - Attributes consumeQueueAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) - .build(); - long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset(); - long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset); - if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) { - measurement.record(0, consumeQueueAttributes); - } else { - measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes); + Attributes commitLogAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) + .build(); + + Attributes consumeQueueAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) + .build(); + long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset(); + long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset); + if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) { + measurement.record(0, consumeQueueAttributes); + } else { + measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes); + } + } catch (ConsumeQueueException e) { + // TODO: handle exception } } }); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 05d88f7b002..db19d344056 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -40,6 +40,7 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:com_alibaba_fastjson2_fastjson2", ], ) @@ -56,7 +57,8 @@ java_library( "//:test_deps", "@maven//:org_apache_commons_commons_lang3", "@maven//:io_netty_netty_all", - "@maven//:commons_cli_commons_cli", + "@maven//:commons_cli_commons_cli", + "@maven//:org_junit_jupiter_junit_jupiter_api", ], resources = glob(["src/test/resources/*.xml"]), ) From 7f8667adde0d33048f03635b31e9a783f297998d Mon Sep 17 00:00:00 2001 From: mawen12 <1181963012mw@gmail.com> Date: Wed, 23 Oct 2024 09:57:35 +0800 Subject: [PATCH 197/265] use correct log method (#8851) --- .../rocketmq/remoting/netty/NettyLogger.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java index 955ffc1bc4f..0a2b2dac595 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyLogger.java @@ -74,7 +74,7 @@ public void log(InternalLogLevel internalLogLevel, String s) { logger.debug(s); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(s); + logger.trace(s); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(s); @@ -93,7 +93,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object o) { logger.debug(s, o); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(s, o); + logger.trace(s, o); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(s, o); @@ -112,7 +112,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object o, Object o1 logger.debug(s, o, o1); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(s, o, o1); + logger.trace(s, o, o1); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(s, o, o1); @@ -131,7 +131,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Object... objects) logger.debug(s, objects); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(s, objects); + logger.trace(s, objects); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(s, objects); @@ -150,7 +150,7 @@ public void log(InternalLogLevel internalLogLevel, String s, Throwable throwable logger.debug(s, throwable); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(s, throwable); + logger.trace(s, throwable); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(s, throwable); @@ -169,7 +169,7 @@ public void log(InternalLogLevel internalLogLevel, Throwable throwable) { logger.debug(EXCEPTION_MESSAGE, throwable); } if (internalLogLevel.equals(InternalLogLevel.TRACE)) { - logger.info(EXCEPTION_MESSAGE, throwable); + logger.trace(EXCEPTION_MESSAGE, throwable); } if (internalLogLevel.equals(InternalLogLevel.INFO)) { logger.info(EXCEPTION_MESSAGE, throwable); @@ -189,32 +189,32 @@ public boolean isTraceEnabled() { @Override public void trace(String var1) { - logger.info(var1); + logger.trace(var1); } @Override public void trace(String var1, Object var2) { - logger.info(var1, var2); + logger.trace(var1, var2); } @Override public void trace(String var1, Object var2, Object var3) { - logger.info(var1, var2, var3); + logger.trace(var1, var2, var3); } @Override public void trace(String var1, Object... var2) { - logger.info(var1, var2); + logger.trace(var1, var2); } @Override public void trace(String var1, Throwable var2) { - logger.info(var1, var2); + logger.trace(var1, var2); } @Override public void trace(Throwable var1) { - logger.info(EXCEPTION_MESSAGE, var1); + logger.trace(EXCEPTION_MESSAGE, var1); } @Override From 2956f6d46b03c5d8b06b06063313386d0a1f68d5 Mon Sep 17 00:00:00 2001 From: Liu Shengzhong Date: Wed, 23 Oct 2024 10:29:56 +0800 Subject: [PATCH 198/265] [ISSUE #8772] Remove lock mq step in broadcasting mode rebalancing (#8774) --- .../apache/rocketmq/client/impl/consumer/RebalanceImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 711df3a9f08..d1f0d116e05 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -308,7 +308,7 @@ private boolean rebalanceByTopic(final String topic, final boolean isOrder) { case BROADCASTING: { Set mqSet = this.topicSubscribeInfoTable.get(topic); if (mqSet != null) { - boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder); + boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, false); if (changed) { this.messageQueueChanged(topic, mqSet, mqSet); log.info("messageQueueChanged {} {} {} {}", consumerGroup, topic, mqSet, mqSet); @@ -477,7 +477,7 @@ private void truncateMessageQueueNotMyTopic() { } private boolean updateProcessQueueTableInRebalance(final String topic, final Set mqSet, - final boolean isOrder) { + final boolean needLockMq) { boolean changed = false; // drop process queues no longer belong me @@ -518,7 +518,7 @@ private boolean updateProcessQueueTableInRebalance(final String topic, final Set List pullRequestList = new ArrayList<>(); for (MessageQueue mq : mqSet) { if (!this.processQueueTable.containsKey(mq)) { - if (isOrder && !this.lock(mq)) { + if (needLockMq && !this.lock(mq)) { log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq); allMQLocked = false; continue; From 738c9f3a952d676353446d72710565b5f8ee4885 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 23 Oct 2024 19:17:05 +0800 Subject: [PATCH 199/265] [ISSUE #8829]feat: add test case to RocksDBConsumeQueueOffsetTable (#8857) * feat: add test case to RocksDBConsumeQueueOffsetTable, verifying forEach works properly for long topic-name Signed-off-by: Li Zhanhui * fix: change OpenJDK distribution from adopt to corretto as the previous one is not updated anymore Signed-off-by: Li Zhanhui * fix: unify set-java version and distribution Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui --- .github/workflows/coverage.yml | 4 +- .github/workflows/integration-test.yml | 4 +- .github/workflows/maven.yaml | 8 +- .github/workflows/snapshot-automation.yml | 8 +- .../queue/RocksDBConsumeQueueOffsetTable.java | 4 +- .../RocksDBConsumeQueueOffsetTableTest.java | 131 ++++++++++++++++++ 6 files changed, 146 insertions(+), 13 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index afa8e0f51ac..b249072f8db 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@master - name: Set up JDK 8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: java-version: "8" - distribution: "adopt" + distribution: "corretto" cache: "maven" - name: Generate coverage report run: mvn -B test -T 2C --file pom.xml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 8179f362879..ad5bcbd6c25 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -26,10 +26,10 @@ jobs: uses: actions/checkout@v2 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} - distribution: "adopt" + distribution: "corretto" cache: "maven" - name: Run integration tests with Maven diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index d0c0ba7d9f1..4eacd65b846 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -19,10 +19,12 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} - distribution: "adopt" + # See https://github.com/actions/setup-java?tab=readme-ov-file#supported-distributions + # AdoptOpenJDK got moved to Eclipse Temurin and won't be updated anymore. + distribution: "corretto" cache: "maven" - name: Build with Maven run: mvn -B package --file pom.xml @@ -41,4 +43,4 @@ jobs: with: name: jvm-crash-logs path: /Users/runner/work/rocketmq/rocketmq/broker/hs_err_pid*.log - retention-days: 1 \ No newline at end of file + retention-days: 1 diff --git a/.github/workflows/snapshot-automation.yml b/.github/workflows/snapshot-automation.yml index 9fb16cb13ca..2c4ede4b6ef 100644 --- a/.github/workflows/snapshot-automation.yml +++ b/.github/workflows/snapshot-automation.yml @@ -59,9 +59,9 @@ jobs: with: ref: ${{ github.event.inputs.branch }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: - distribution: "temurin" + distribution: "corretto" java-version: "8" cache: "maven" - name: Build distribution tar @@ -238,10 +238,10 @@ jobs: ref: develop persist-credentials: false - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: 8 - distribution: "temurin" + distribution: "corretto" cache: "maven" - name: Update default pom version if: github.event.inputs.rocketmq_version == '' diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java index 889131d1cc8..a91ae5e244e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java @@ -85,7 +85,7 @@ public class RocksDBConsumeQueueOffsetTable { * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤ */ - private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; + public static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; private static final int OFFSET_VALUE_LENGTH = 8 + 8; /** @@ -682,7 +682,7 @@ private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, fina return byteBuffer; } - private static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, + public static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java new file mode 100644 index 00000000000..b1e12d49468 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTableTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.queue.offset.OffsetEntryType; +import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +@RunWith(MockitoJUnitRunner.class) +public class RocksDBConsumeQueueOffsetTableTest { + + private RocksDBConsumeQueueOffsetTable offsetTable; + + @Mock + private ConsumeQueueRocksDBStorage rocksDBStorage; + + @Mock + private RocksDBConsumeQueueTable consumeQueueTable; + + @Mock + private DefaultMessageStore messageStore; + + private static RocksDB db; + + private static File dbPath; + + private static String topicName; + + @BeforeClass + public static void initDB() throws IOException, RocksDBException { + TemporaryFolder tempFolder = new TemporaryFolder(); + tempFolder.create(); + dbPath = tempFolder.newFolder(); + + db = RocksDB.open(dbPath.getAbsolutePath()); + StringBuilder topicBuilder = new StringBuilder(); + for (int i = 0; i < 100; i++) { + topicBuilder.append("topic"); + } + topicName = topicBuilder.toString(); + byte[] topicInBytes = topicName.getBytes(StandardCharsets.UTF_8); + + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicInBytes.length); + RocksDBConsumeQueueOffsetTable.buildOffsetKeyByteBuffer(keyBuffer, topicInBytes, 1, true); + Assert.assertEquals(0, keyBuffer.position()); + Assert.assertEquals(RocksDBConsumeQueueOffsetTable.OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicInBytes.length, keyBuffer.limit()); + + ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES + Long.BYTES); + valueBuffer.putLong(100); + valueBuffer.putLong(2); + valueBuffer.flip(); + + try (WriteBatch writeBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions()) { + writeOptions.setDisableWAL(false); + writeOptions.setSync(true); + writeBatch.put(keyBuffer, valueBuffer); + db.write(writeOptions, writeBatch); + } + + } + + @AfterClass + public static void tearDownDB() throws RocksDBException { + db.closeE(); + RocksDB.destroyDB(dbPath.getAbsolutePath(), new Options()); + } + + @Before + public void setUp() { + RocksIterator iterator = db.newIterator(); + Mockito.doReturn(iterator).when(rocksDBStorage).seekOffsetCF(); + offsetTable = new RocksDBConsumeQueueOffsetTable(consumeQueueTable, rocksDBStorage, messageStore); + } + + /** + * Verify forEach can expand key-buffer properly and works well for long topic names. + * + * @throws RocksDBException If there is an RocksDB error. + */ + @Test + public void testForEach() throws RocksDBException { + AtomicBoolean called = new AtomicBoolean(false); + offsetTable.forEach(entry -> true, entry -> { + called.set(true); + Assert.assertEquals(topicName, entry.topic); + Assert.assertTrue(topicName.length() > 256); + Assert.assertEquals(1, entry.queueId); + Assert.assertEquals(100, entry.commitLogOffset); + Assert.assertEquals(2, entry.offset); + Assert.assertEquals(OffsetEntryType.MAXIMUM, entry.type); + }); + Assert.assertTrue(called.get()); + } +} From 95b88ff8fcfecbd1942e1a35460f7417ee620673 Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:17:37 +0800 Subject: [PATCH 200/265] [ISSUE #8442][RIP-70-3] Extract adaptive lock mechanism (#8663) * extract the adaptive lock * extract the adaptive lock * feat(): perfect the adaptive lock * feat(): perfect the adaptive lock * Optimized code type * Optimized code type * Optimized code type * fix fail test * Optimize the adaptive locking mechanism logic * Optimize the adaptive locking mechanism logic * feat:Adaptive locking mechanism adjustment * feat:Adaptive locking mechanism adjustment * feat:Adaptive locking mechanism adjustment * Optimize the adaptive locking mechanism logic * Optimize the adaptive locking mechanism logic * Optimize the adaptive locking mechanism logic * feat:Supports the hot activation of ABS locks * feat:Supports the hot activation of ABS locks * feat:Supports the hot activation of ABS locks * feat:Supports the hot activation of ABS locks * Optimize code style * Optimize code style * Optimize code style * Optimize code style * Optimize code style * Optimize code style * Updated the locking mechanism name * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * delete unused import * Optimize the logic of switching to spin locks * Revert "Optimize the logic of switching to spin locks" This reverts commit 1d7bac5c2fea0531af01d4c57c843084ba4fea61. * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimize the logic of switching to spin locks * Optimized locking logic * Optimized locking logic * Optimized locking logic * fix test * fix test * fix test * fix test * Optimize code style * Optimize code style * fix test * fix test * optimize client rebalancing logic --------- Co-authored-by: wanghuaiyuan --- .../org/apache/rocketmq/store/CommitLog.java | 7 +- .../store/config/MessageStoreConfig.java | 27 +++ .../store/lock/AdaptiveBackOffSpinLock.java | 35 +++ .../lock/AdaptiveBackOffSpinLockImpl.java | 207 ++++++++++++++++++ .../store/lock/BackOffReentrantLock.java | 33 +++ .../rocketmq/store/lock/BackOffSpinLock.java | 110 ++++++++++ .../rocketmq/store/lock/AdaptiveLockTest.java | 86 ++++++++ 7 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java create mode 100644 store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 153215c98ad..63022520e2a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -62,6 +62,7 @@ import org.apache.rocketmq.store.exception.StoreException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; +import org.apache.rocketmq.store.lock.AdaptiveBackOffSpinLockImpl; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -130,7 +131,11 @@ protected PutMessageThreadLocal initialValue() { return new PutMessageThreadLocal(defaultMessageStore.getMessageStoreConfig()); } }; - this.putMessageLock = messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); + + PutMessageLock adaptiveBackOffSpinLock = new AdaptiveBackOffSpinLockImpl(); + + this.putMessageLock = messageStore.getMessageStoreConfig().getUseABSLock() ? adaptiveBackOffSpinLock : + messageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage() ? new PutMessageReentrantLock() : new PutMessageSpinLock(); this.flushDiskWatcher = new FlushDiskWatcher(); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 8effe35bab6..e31c03dd22b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -445,6 +445,17 @@ public class MessageStoreConfig { */ private String bottomMostCompressionTypeForConsumeQueueStore = "zstd"; + /** + * Spin number in the retreat strategy of spin lock + * Default is 1000 + */ + private int spinLockCollisionRetreatOptimalDegree = 1000; + + /** + * Use AdaptiveBackOffLock + **/ + private boolean useABSLock = false; + public boolean isRocksdbCQDoubleWriteEnable() { return rocksdbCQDoubleWriteEnable; } @@ -1898,4 +1909,20 @@ public String getBottomMostCompressionTypeForConsumeQueueStore() { public void setBottomMostCompressionTypeForConsumeQueueStore(String bottomMostCompressionTypeForConsumeQueueStore) { this.bottomMostCompressionTypeForConsumeQueueStore = bottomMostCompressionTypeForConsumeQueueStore; } + + public int getSpinLockCollisionRetreatOptimalDegree() { + return spinLockCollisionRetreatOptimalDegree; + } + + public void setSpinLockCollisionRetreatOptimalDegree(int spinLockCollisionRetreatOptimalDegree) { + this.spinLockCollisionRetreatOptimalDegree = spinLockCollisionRetreatOptimalDegree; + } + + public void setUseABSLock(boolean useABSLock) { + this.useABSLock = useABSLock; + } + + public boolean getUseABSLock() { + return useABSLock; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java new file mode 100644 index 00000000000..96200bcc15b --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLock.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.PutMessageLock; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +public interface AdaptiveBackOffSpinLock extends PutMessageLock { + /** + * Configuration update + * @param messageStoreConfig + */ + default void update(MessageStoreConfig messageStoreConfig) { + } + + /** + * Locking mechanism switching + */ + default void swap() { + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java new file mode 100644 index 00000000000..b4abb082718 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/AdaptiveBackOffSpinLockImpl.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.config.MessageStoreConfig; + +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class AdaptiveBackOffSpinLockImpl implements AdaptiveBackOffSpinLock { + private AdaptiveBackOffSpinLock adaptiveLock; + //state + private AtomicBoolean state = new AtomicBoolean(true); + + // Used to determine the switchover between a mutex lock and a spin lock + private final static float SWAP_SPIN_LOCK_RATIO = 0.8f; + + // It is used to adjust the spin number K of the escape spin lock + // When (retreat number / TPS) <= (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO), K is decreased + private final static int SPIN_LOCK_ADAPTIVE_RATIO = 4; + + // It is used to adjust the spin number K of the escape spin lock + // When (retreat number / TPS) >= (1 / BASE_SWAP_ADAPTIVE_RATIO), K is increased + private final static int BASE_SWAP_LOCK_RATIO = 320; + + private final static String BACK_OFF_SPIN_LOCK = "SpinLock"; + + private final static String REENTRANT_LOCK = "ReentrantLock"; + + private Map locks; + + private final List tpsTable; + + private final List> threadTable; + + private int swapCriticalPoint; + + private AtomicInteger currentThreadNum = new AtomicInteger(0); + + private AtomicBoolean isOpen = new AtomicBoolean(true); + + public AdaptiveBackOffSpinLockImpl() { + this.locks = new HashMap<>(); + this.locks.put(REENTRANT_LOCK, new BackOffReentrantLock()); + this.locks.put(BACK_OFF_SPIN_LOCK, new BackOffSpinLock()); + + this.threadTable = new ArrayList<>(2); + this.threadTable.add(new ConcurrentHashMap<>()); + this.threadTable.add(new ConcurrentHashMap<>()); + + this.tpsTable = new ArrayList<>(2); + this.tpsTable.add(new AtomicInteger(0)); + this.tpsTable.add(new AtomicInteger(0)); + + adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK); + } + + @Override + public void lock() { + int slot = LocalTime.now().getSecond() % 2; + this.threadTable.get(slot).putIfAbsent(Thread.currentThread(), Byte.MAX_VALUE); + this.tpsTable.get(slot).getAndIncrement(); + boolean state; + do { + state = this.state.get(); + } while (!state); + + currentThreadNum.incrementAndGet(); + this.adaptiveLock.lock(); + } + + @Override + public void unlock() { + this.adaptiveLock.unlock(); + currentThreadNum.decrementAndGet(); + if (isOpen.get()) { + swap(); + } + } + + @Override + public void update(MessageStoreConfig messageStoreConfig) { + this.adaptiveLock.update(messageStoreConfig); + } + + @Override + public void swap() { + if (!this.state.get()) { + return; + } + boolean needSwap = false; + int slot = 1 - LocalTime.now().getSecond() % 2; + int tps = this.tpsTable.get(slot).get() + 1; + int threadNum = this.threadTable.get(slot).size(); + this.tpsTable.get(slot).set(-1); + this.threadTable.get(slot).clear(); + if (tps == 0) { + return; + } + + if (this.adaptiveLock instanceof BackOffSpinLock) { + BackOffSpinLock lock = (BackOffSpinLock) this.adaptiveLock; + // Avoid frequent adjustment of K, and make a reasonable range through experiments + // reasonable range : (retreat number / TPS) > (1 / BASE_SWAP_ADAPTIVE_RATIO * SPIN_LOCK_ADAPTIVE_RATIO) && + // (retreat number / TPS) < (1 / BASE_SWAP_ADAPTIVE_RATIO) + if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO >= tps) { + if (lock.isAdapt()) { + lock.adapt(true); + } else { + // It is used to switch between mutex lock and spin lock + this.swapCriticalPoint = tps * threadNum; + needSwap = true; + } + } else if (lock.getNumberOfRetreat(slot) * BASE_SWAP_LOCK_RATIO * SPIN_LOCK_ADAPTIVE_RATIO <= tps) { + lock.adapt(false); + } + lock.setNumberOfRetreat(slot, 0); + } else { + if (tps * threadNum <= this.swapCriticalPoint * SWAP_SPIN_LOCK_RATIO) { + needSwap = true; + } + } + + if (needSwap) { + if (this.state.compareAndSet(true, false)) { + // Ensures that no threads are in contention locks as well as in critical zones + int currentThreadNum; + do { + currentThreadNum = this.currentThreadNum.get(); + } while (currentThreadNum != 0); + + try { + if (this.adaptiveLock instanceof BackOffSpinLock) { + this.adaptiveLock = this.locks.get(REENTRANT_LOCK); + } else { + this.adaptiveLock = this.locks.get(BACK_OFF_SPIN_LOCK); + ((BackOffSpinLock) this.adaptiveLock).adapt(false); + } + } catch (Exception e) { + //ignore + } finally { + this.state.compareAndSet(false, true); + } + } + } + } + + public List getLocks() { + return (List) this.locks.values(); + } + + public void setLocks(Map locks) { + this.locks = locks; + } + + public boolean getState() { + return this.state.get(); + } + + public void setState(boolean state) { + this.state.set(state); + } + + public AdaptiveBackOffSpinLock getAdaptiveLock() { + return adaptiveLock; + } + + public List getTpsTable() { + return tpsTable; + } + + public void setSwapCriticalPoint(int swapCriticalPoint) { + this.swapCriticalPoint = swapCriticalPoint; + } + + public int getSwapCriticalPoint() { + return swapCriticalPoint; + } + + public boolean isOpen() { + return this.isOpen.get(); + } + + public void setOpen(boolean open) { + this.isOpen.set(open); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java new file mode 100644 index 00000000000..90e416419bc --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffReentrantLock.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import java.util.concurrent.locks.ReentrantLock; + +public class BackOffReentrantLock implements AdaptiveBackOffSpinLock { + private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync + + @Override + public void lock() { + putMessageNormalLock.lock(); + } + + @Override + public void unlock() { + putMessageNormalLock.unlock(); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java new file mode 100644 index 00000000000..f754970a054 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/lock/BackOffSpinLock.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.apache.rocketmq.store.config.MessageStoreConfig; + +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class BackOffSpinLock implements AdaptiveBackOffSpinLock { + + private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); + + private int optimalDegree; + + private final static int INITIAL_DEGREE = 1000; + + private final static int MAX_OPTIMAL_DEGREE = 10000; + + private final List numberOfRetreat; + + public BackOffSpinLock() { + this.optimalDegree = INITIAL_DEGREE; + + numberOfRetreat = new ArrayList<>(2); + numberOfRetreat.add(new AtomicInteger(0)); + numberOfRetreat.add(new AtomicInteger(0)); + } + + @Override + public void lock() { + int spinDegree = this.optimalDegree; + while (true) { + for (int i = 0; i < spinDegree; i++) { + if (this.putMessageSpinLock.compareAndSet(true, false)) { + return; + } + } + numberOfRetreat.get(LocalTime.now().getSecond() % 2).getAndIncrement(); + try { + Thread.sleep(0); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + @Override + public void unlock() { + this.putMessageSpinLock.compareAndSet(false, true); + } + + @Override + public void update(MessageStoreConfig messageStoreConfig) { + this.optimalDegree = messageStoreConfig.getSpinLockCollisionRetreatOptimalDegree(); + } + + public int getOptimalDegree() { + return this.optimalDegree; + } + + public void setOptimalDegree(int optimalDegree) { + this.optimalDegree = optimalDegree; + } + + public boolean isAdapt() { + return optimalDegree < MAX_OPTIMAL_DEGREE; + } + + public synchronized void adapt(boolean isRise) { + if (isRise) { + if (optimalDegree * 2 <= MAX_OPTIMAL_DEGREE) { + optimalDegree *= 2; + } else { + if (optimalDegree + INITIAL_DEGREE <= MAX_OPTIMAL_DEGREE) { + optimalDegree += INITIAL_DEGREE; + } + } + } else { + if (optimalDegree >= 2 * INITIAL_DEGREE) { + optimalDegree -= INITIAL_DEGREE; + } + } + } + + public int getNumberOfRetreat(int pos) { + return numberOfRetreat.get(pos).get(); + } + + public void setNumberOfRetreat(int pos, int size) { + this.numberOfRetreat.get(pos).set(size); + } +} diff --git a/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java new file mode 100644 index 00000000000..ac1b3c60cc0 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/lock/AdaptiveLockTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.lock; + +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class AdaptiveLockTest { + + AdaptiveBackOffSpinLockImpl adaptiveLock; + + @Before + public void init() { + adaptiveLock = new AdaptiveBackOffSpinLockImpl(); + } + + @Test + public void testAdaptiveLock() throws InterruptedException { + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); + CountDownLatch countDownLatch = new CountDownLatch(1); + adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //ignore + } + adaptiveLock.unlock(); + countDownLatch.countDown(); + } + }).start(); + Thread.sleep(1000); + adaptiveLock.unlock(); + assertEquals(2000, ((BackOffSpinLock) adaptiveLock.getAdaptiveLock()).getOptimalDegree()); + countDownLatch.await(); + + for (int i = 0; i <= 5; i++) { + CountDownLatch countDownLatch1 = new CountDownLatch(1); + adaptiveLock.lock(); + new Thread(new Runnable() { + @Override + public void run() { + adaptiveLock.lock(); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //ignore + } + adaptiveLock.unlock(); + countDownLatch1.countDown(); + } + }).start(); + Thread.sleep(1000); + adaptiveLock.unlock(); + countDownLatch1.await(); + } + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffReentrantLock); + + adaptiveLock.lock(); + Thread.sleep(1000); + adaptiveLock.unlock(); + assertTrue(adaptiveLock.getAdaptiveLock() instanceof BackOffSpinLock); + } +} From c8938e0c6c263cdb73b593ee9984841115604679 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Mon, 28 Oct 2024 10:00:33 +0800 Subject: [PATCH 201/265] [ISSUE #8829] feat: provide ConfigManagerV2 to make best uses of RocksDB (#8856) * feat: provide ConfigManagerV2 to make best uses of RocksDB Signed-off-by: Li Zhanhui * fix: release RocksDB objects using try-with-resource Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui --- .../rocketmq/broker/BrokerController.java | 39 +- .../v1}/RocksDBConsumerOffsetManager.java | 3 +- .../v1}/RocksDBLmqConsumerOffsetManager.java | 2 +- .../RocksDBLmqSubscriptionGroupManager.java | 2 +- .../v1}/RocksDBLmqTopicConfigManager.java | 2 +- .../v1}/RocksDBOffsetSerializeWrapper.java | 2 +- .../v1}/RocksDBSubscriptionGroupManager.java | 3 +- .../v1}/RocksDBTopicConfigManager.java | 3 +- .../broker/config/v2/ConfigHelper.java | 132 ++++++ .../broker/config/v2/ConfigStorage.java | 122 +++++ .../config/v2/ConsumerOffsetManagerV2.java | 426 ++++++++++++++++++ .../broker/config/v2/RecordPrefix.java | 33 ++ .../broker/config/v2/SerializationType.java | 46 ++ .../config/v2/SubscriptionGroupManagerV2.java | 171 +++++++ .../rocketmq/broker/config/v2/TableId.java | 38 ++ .../broker/config/v2/TablePrefix.java | 32 ++ .../config/v2/TopicConfigManagerV2.java | 191 ++++++++ .../broker/config/v2/package-info.java | 26 ++ .../broker/offset/ConsumerOffsetManager.java | 2 +- .../SubscriptionGroupManager.java | 4 +- .../broker/topic/TopicConfigManager.java | 4 +- .../RocksDBConsumerOffsetManagerTest.java | 1 + .../RocksDBLmqConsumerOffsetManagerTest.java | 1 + .../RocksDBOffsetSerializeWrapperTest.java | 1 + .../RocksdbTransferOffsetAndCqTest.java | 1 + .../processor/AdminBrokerProcessorTest.java | 4 +- .../RocksdbGroupConfigTransferTest.java | 1 + .../topic/RocksdbTopicConfigManagerTest.java | 1 + .../topic/RocksdbTopicConfigTransferTest.java | 1 + .../apache/rocketmq/common/BrokerConfig.java | 14 + .../common/config/ConfigManagerVersion.java | 33 ++ 31 files changed, 1319 insertions(+), 22 deletions(-) rename broker/src/main/java/org/apache/rocketmq/broker/{offset => config/v1}/RocksDBConsumerOffsetManager.java (98%) rename broker/src/main/java/org/apache/rocketmq/broker/{offset => config/v1}/RocksDBLmqConsumerOffsetManager.java (98%) rename broker/src/main/java/org/apache/rocketmq/broker/{subscription => config/v1}/RocksDBLmqSubscriptionGroupManager.java (97%) rename broker/src/main/java/org/apache/rocketmq/broker/{topic => config/v1}/RocksDBLmqTopicConfigManager.java (97%) rename broker/src/main/java/org/apache/rocketmq/broker/{offset => config/v1}/RocksDBOffsetSerializeWrapper.java (96%) rename broker/src/main/java/org/apache/rocketmq/broker/{subscription => config/v1}/RocksDBSubscriptionGroupManager.java (98%) rename broker/src/main/java/org/apache/rocketmq/broker/{topic => config/v1}/RocksDBTopicConfigManager.java (98%) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 05a00a50053..ee211e1b80a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -76,8 +76,8 @@ import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager; import org.apache.rocketmq.broker.offset.LmqConsumerOffsetManager; -import org.apache.rocketmq.broker.offset.RocksDBConsumerOffsetManager; -import org.apache.rocketmq.broker.offset.RocksDBLmqConsumerOffsetManager; +import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; +import org.apache.rocketmq.broker.config.v1.RocksDBLmqConsumerOffsetManager; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin; import org.apache.rocketmq.broker.processor.AckMessageProcessor; @@ -99,12 +99,16 @@ import org.apache.rocketmq.broker.schedule.ScheduleMessageService; import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.LmqSubscriptionGroupManager; -import org.apache.rocketmq.broker.subscription.RocksDBLmqSubscriptionGroupManager; -import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager; +import org.apache.rocketmq.broker.config.v1.RocksDBLmqSubscriptionGroupManager; +import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; +import org.apache.rocketmq.broker.config.v2.ConsumerOffsetManagerV2; +import org.apache.rocketmq.broker.config.v2.SubscriptionGroupManagerV2; +import org.apache.rocketmq.broker.config.v2.TopicConfigManagerV2; +import org.apache.rocketmq.broker.config.v2.ConfigStorage; import org.apache.rocketmq.broker.topic.LmqTopicConfigManager; -import org.apache.rocketmq.broker.topic.RocksDBLmqTopicConfigManager; -import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager; +import org.apache.rocketmq.broker.config.v1.RocksDBLmqTopicConfigManager; +import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService; import org.apache.rocketmq.broker.topic.TopicQueueMappingManager; @@ -124,6 +128,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.ConfigManagerVersion; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageExt; @@ -239,6 +244,11 @@ public class BrokerController { protected RemotingServer remotingServer; protected CountDownLatch remotingServerStartLatch; protected RemotingServer fastRemotingServer; + + /** + * If {Topic, SubscriptionGroup, Offset}ManagerV2 are used, config entries are stored in RocksDB. + */ + protected ConfigStorage configStorage; protected TopicConfigManager topicConfigManager; protected SubscriptionGroupManager subscriptionGroupManager; protected TopicQueueMappingManager topicQueueMappingManager; @@ -334,7 +344,12 @@ public BrokerController( this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); this.broadcastOffsetManager = new BroadcastOffsetManager(this); - if (this.messageStoreConfig.isEnableRocksDBStore()) { + if (ConfigManagerVersion.V2.getVersion().equals(brokerConfig.getConfigManagerVersion())) { + this.configStorage = new ConfigStorage(messageStoreConfig.getStorePathRootDir()); + this.topicConfigManager = new TopicConfigManagerV2(this, configStorage); + this.subscriptionGroupManager = new SubscriptionGroupManagerV2(this, configStorage); + this.consumerOffsetManager = new ConsumerOffsetManagerV2(this, configStorage); + } else if (this.messageStoreConfig.isEnableRocksDBStore()) { this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); @@ -771,7 +786,11 @@ private void updateNamesrvAddr() { } public boolean initializeMetadata() { - boolean result = this.topicConfigManager.load(); + boolean result = true; + if (null != configStorage) { + result = configStorage.start(); + } + result = result && this.topicConfigManager.load(); result = result && this.topicQueueMappingManager.load(); result = result && this.consumerOffsetManager.load(); result = result && this.subscriptionGroupManager.load(); @@ -1547,6 +1566,10 @@ protected void shutdownBasicService() { this.consumerOffsetManager.stop(); } + if (null != configStorage) { + configStorage.shutdown(); + } + if (this.authenticationMetadataManager != null) { this.authenticationMetadataManager.shutdown(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java similarity index 98% rename from broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java index 1e7cda71eed..8066fe769a0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.offset; +package org.apache.rocketmq.broker.config.v1; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.DataConverter; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java similarity index 98% rename from broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java index d0faa661406..e961c6c635a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.offset; +package org.apache.rocketmq.broker.config.v1; import java.util.HashMap; import java.util.Map; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java similarity index 97% rename from broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java index e4de25756bf..05f3f7d2ec3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBLmqSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqSubscriptionGroupManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.subscription; +package org.apache.rocketmq.broker.config.v1; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.MixAll; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java similarity index 97% rename from broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java index d049a8dbcde..7b278013965 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBLmqTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqTopicConfigManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.topic; +package org.apache.rocketmq.broker.config.v1; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.MixAll; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java similarity index 96% rename from broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java index 7a90fd62fb8..4801cfc681c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapper.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBOffsetSerializeWrapper.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.offset; +package org.apache.rocketmq.broker.config.v1; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java similarity index 98% rename from broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java index 5119f78672c..8175d63cce1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.subscription; +package org.apache.rocketmq.broker.config.v1; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; @@ -27,6 +27,7 @@ import java.util.function.BiConsumer; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.protocol.DataVersion; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java similarity index 98% rename from broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java rename to broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java index 466e6416f98..bce67392f64 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.topic; +package org.apache.rocketmq.broker.config.v1; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.RocksDBConfigManager; +import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.utils.DataConverter; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java new file mode 100644 index 00000000000..8183a1f8358 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +import com.alibaba.fastjson2.JSON; +import com.google.common.base.Preconditions; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; + +public class ConfigHelper { + + /** + *

    + * Layout of data version key: + * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes] + *

    + * + *

    + * Layout of data version value: + * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes] + *

    + * + * @throws RocksDBException if RocksDB raises an error + */ + public static Optional loadDataVersion(ConfigStorage configStorage, TableId tableId) + throws RocksDBException { + int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */ + + ConfigStorage.DATA_VERSION_KEY_BYTES.length; + ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + try { + keyBuf.writeByte(TablePrefix.TABLE.getValue()); + keyBuf.writeShort(tableId.getValue()); + keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue()); + keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES); + byte[] valueByes = configStorage.get(keyBuf.nioBuffer()); + if (null != valueByes) { + ByteBuf valueBuf = Unpooled.wrappedBuffer(valueByes); + return Optional.of(valueBuf); + } + } finally { + keyBuf.release(); + } + return Optional.empty(); + } + + public static void stampDataVersion(WriteBatch writeBatch, DataVersion dataVersion, long stateMachineVersion) + throws RocksDBException { + // Increase data version + dataVersion.nextVersion(stateMachineVersion); + + int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */ + + ConfigStorage.DATA_VERSION_KEY_BYTES.length; + ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(Long.BYTES * 3); + try { + keyBuf.writeByte(TablePrefix.TABLE.getValue()); + keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue()); + keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue()); + keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES); + valueBuf.writeLong(dataVersion.getStateVersion()); + valueBuf.writeLong(dataVersion.getTimestamp()); + valueBuf.writeLong(dataVersion.getCounter().get()); + writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); + } finally { + keyBuf.release(); + valueBuf.release(); + } + } + + public static void onDataVersionLoad(ByteBuf buf, DataVersion dataVersion) { + if (buf.readableBytes() == 8 /* state machine version */ + 8 /* timestamp */ + 8 /* counter */) { + long stateMachineVersion = buf.readLong(); + long timestamp = buf.readLong(); + long counter = buf.readLong(); + dataVersion.setStateVersion(stateMachineVersion); + dataVersion.setTimestamp(timestamp); + dataVersion.setCounter(new AtomicLong(counter)); + } + buf.release(); + } + + public static ByteBuf keyBufOf(TableId tableId, final String name) { + Preconditions.checkNotNull(name); + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */ + 2 /* name-length */ + bytes.length; + ByteBuf keyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + keyBuf.writeByte(TablePrefix.TABLE.getValue()); + keyBuf.writeShort(tableId.getValue()); + keyBuf.writeByte(RecordPrefix.DATA.getValue()); + keyBuf.writeShort(bytes.length); + keyBuf.writeBytes(bytes); + return keyBuf; + } + + public static ByteBuf valueBufOf(final Object config, SerializationType serializationType) { + if (SerializationType.JSON == serializationType) { + byte[] payload = JSON.toJSONBytes(config); + ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(1 + payload.length); + valueBuf.writeByte(SerializationType.JSON.getValue()); + valueBuf.writeBytes(payload); + return valueBuf; + } + throw new RuntimeException("Unsupported serialization type: " + serializationType); + } + + public static byte[] readBytes(final ByteBuf buf) { + byte[] bytes = new byte[buf.readableBytes()]; + buf.readBytes(bytes); + return bytes; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java new file mode 100644 index 00000000000..af259aaa374 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +import io.netty.util.internal.PlatformDependent; +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.common.config.ConfigHelper; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DirectSlice; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; + +/** + * https://book.tidb.io/session1/chapter3/tidb-kv-to-relation.html + */ +public class ConfigStorage extends AbstractRocksDBStorage { + + public static final String DATA_VERSION_KEY = "data_version"; + public static final byte[] DATA_VERSION_KEY_BYTES = DATA_VERSION_KEY.getBytes(StandardCharsets.UTF_8); + + public ConfigStorage(String storePath) { + super(storePath + File.separator + "config" + File.separator + "rdb"); + } + + @Override + protected boolean postLoad() { + if (!PlatformDependent.hasUnsafe()) { + LOGGER.error("Unsafe not available and POOLED_ALLOCATOR cannot work correctly"); + return false; + } + try { + UtilAll.ensureDirOK(this.dbPath); + initOptions(); + List cfDescriptors = new ArrayList<>(); + + ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigOptions(); + this.cfOptions.add(defaultOptions); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); + + // Start RocksDB instance + open(cfDescriptors); + + this.defaultCFHandle = cfHandles.get(0); + } catch (final Exception e) { + AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e); + return false; + } + return true; + } + + @Override + protected void preShutdown() { + + } + + protected void initOptions() { + this.options = ConfigHelper.createConfigDBOptions(); + super.initOptions(); + } + + @Override + protected void initAbleWalWriteOptions() { + this.ableWalWriteOptions = new WriteOptions(); + + // For metadata, prioritize data integrity + this.ableWalWriteOptions.setSync(true); + + // We need WAL for config changes + this.ableWalWriteOptions.setDisableWAL(false); + + // No fast failure on block, wait synchronously even if there is wait for the write request + this.ableWalWriteOptions.setNoSlowdown(false); + } + + public byte[] get(ByteBuffer key) throws RocksDBException { + byte[] keyBytes = new byte[key.remaining()]; + key.get(keyBytes); + return super.get(getDefaultCFHandle(), totalOrderReadOptions, keyBytes); + } + + public void write(WriteBatch writeBatch) throws RocksDBException { + db.write(ableWalWriteOptions, writeBatch); + } + + public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) { + try (ReadOptions readOptions = new ReadOptions()) { + readOptions.setTotalOrderSeek(true); + readOptions.setTailing(false); + readOptions.setAutoPrefixMode(true); + readOptions.setIterateLowerBound(new DirectSlice(beginKey)); + readOptions.setIterateUpperBound(new DirectSlice(endKey)); + RocksIterator iterator = db.newIterator(defaultCFHandle, readOptions); + iterator.seekToFirst(); + return iterator; + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java new file mode 100644 index 00000000000..5b0885c491a --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +import io.netty.buffer.ByteBuf; +import io.netty.util.internal.PlatformDependent; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.store.MessageStore; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; + +/** + *

    + * Layout of consumer offset key: + * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte] + * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes] + *

    + * + *

    + * Layout of consumer offset value: [offset, 8 bytes] + *

    + */ +public class ConsumerOffsetManagerV2 extends ConsumerOffsetManager { + + private final ConfigStorage configStorage; + + public ConsumerOffsetManagerV2(BrokerController brokerController, ConfigStorage configStorage) { + super(brokerController); + this.configStorage = configStorage; + } + + @Override + protected void removeConsumerOffset(String topicAtGroup) { + if (!MixAll.isLmq(topicAtGroup)) { + super.removeConsumerOffset(topicAtGroup); + } + + String[] topicGroup = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (topicGroup.length != 2) { + LOG.error("Invalid topic group: {}", topicAtGroup); + return; + } + + byte[] topicBytes = topicGroup[0].getBytes(StandardCharsets.UTF_8); + byte[] groupBytes = topicGroup[1].getBytes(StandardCharsets.UTF_8); + + int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */ + + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */ + + Short.BYTES + topicBytes.length + 1; + // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group-bytes][CTRL_1, 1 byte] + // [topic-len, 2 bytes][topic-bytes][CTRL_1] + ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + beginKey.writeByte(TablePrefix.TABLE.getValue()); + beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue()); + beginKey.writeByte(RecordPrefix.DATA.getValue()); + beginKey.writeShort(groupBytes.length); + beginKey.writeBytes(groupBytes); + beginKey.writeByte(AbstractRocksDBStorage.CTRL_1); + beginKey.writeShort(topicBytes.length); + beginKey.writeBytes(topicBytes); + beginKey.writeByte(AbstractRocksDBStorage.CTRL_1); + + ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + endKey.writeByte(TablePrefix.TABLE.getValue()); + endKey.writeShort(TableId.CONSUMER_OFFSET.getValue()); + endKey.writeByte(RecordPrefix.DATA.getValue()); + endKey.writeShort(groupBytes.length); + endKey.writeBytes(groupBytes); + endKey.writeByte(AbstractRocksDBStorage.CTRL_1); + endKey.writeShort(topicBytes.length); + endKey.writeBytes(topicBytes); + endKey.writeByte(AbstractRocksDBStorage.CTRL_2); + + try (WriteBatch writeBatch = new WriteBatch()) { + // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here + writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey)); + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + LOG.error("Failed to removeConsumerOffset, topicAtGroup={}", topicAtGroup, e); + } finally { + beginKey.release(); + endKey.release(); + } + } + + @Override + public void removeOffset(String group) { + if (!MixAll.isLmq(group)) { + super.removeOffset(group); + } + + byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8); + int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */ + + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */; + + // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte] + ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + beginKey.writeByte(TablePrefix.TABLE.getValue()); + beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue()); + beginKey.writeByte(RecordPrefix.DATA.getValue()); + beginKey.writeShort(groupBytes.length); + beginKey.writeBytes(groupBytes); + beginKey.writeByte(AbstractRocksDBStorage.CTRL_1); + + ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + endKey.writeByte(TablePrefix.TABLE.getValue()); + endKey.writeShort(TableId.CONSUMER_OFFSET.getValue()); + endKey.writeByte(RecordPrefix.DATA.getValue()); + endKey.writeShort(groupBytes.length); + endKey.writeBytes(groupBytes); + endKey.writeByte(AbstractRocksDBStorage.CTRL_2); + try (WriteBatch writeBatch = new WriteBatch()) { + // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here + writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey)); + MessageStore messageStore = brokerController.getMessageStore(); + long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + LOG.error("Failed to consumer offsets by group={}", group, e); + } finally { + beginKey.release(); + endKey.release(); + } + } + + /** + *

    + * Layout of consumer offset key: + * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte] + * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes] + *

    + * + *

    + * Layout of consumer offset value: + * [offset, 8 bytes] + *

    + * + * @param clientHost The client that submits consumer offsets + * @param group Group name + * @param topic Topic name + * @param queueId Queue ID + * @param offset Consumer offset of the specified queue + */ + @Override + public void commitOffset(String clientHost, String group, String topic, int queueId, long offset) { + String key = topic + TOPIC_GROUP_SEPARATOR + group; + + // We maintain a copy of classic consumer offset table in memory as they take very limited memory footprint. + // For LMQ offsets, given the volume and number of these type of records, they are maintained in RocksDB + // directly. Frequently used LMQ consumer offsets should reside either in block-cache or MemTable, so read/write + // should be blazingly fast. + if (!MixAll.isLmq(topic)) { + if (offsetTable.containsKey(key)) { + offsetTable.get(key).put(queueId, offset); + } else { + ConcurrentMap map = new ConcurrentHashMap<>(); + ConcurrentMap prev = offsetTable.putIfAbsent(key, map); + if (null != prev) { + map = prev; + } + map.put(queueId, offset); + } + } + + ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId); + ByteBuf valueBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(Long.BYTES); + try (WriteBatch writeBatch = new WriteBatch()) { + valueBuf.writeLong(offset); + writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); + MessageStore messageStore = brokerController.getMessageStore(); + long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + LOG.error("Failed to commit consumer offset", e); + } finally { + keyBuf.release(); + valueBuf.release(); + } + } + + private ByteBuf keyOfConsumerOffset(String group, String topic, int queueId) { + byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8); + byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); + int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/ + + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/ + + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/ + + Integer.BYTES /*queue-id*/; + ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen); + keyBuf.writeByte(TablePrefix.TABLE.getValue()); + keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue()); + keyBuf.writeByte(RecordPrefix.DATA.getValue()); + keyBuf.writeShort(groupBytes.length); + keyBuf.writeBytes(groupBytes); + keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1); + keyBuf.writeShort(topicBytes.length); + keyBuf.writeBytes(topicBytes); + keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1); + keyBuf.writeInt(queueId); + return keyBuf; + } + + private ByteBuf keyOfPullOffset(String group, String topic, int queueId) { + byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8); + byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); + int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/ + + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/ + + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/ + + Integer.BYTES /*queue-id*/; + ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen); + keyBuf.writeByte(TablePrefix.TABLE.getValue()); + keyBuf.writeShort(TableId.PULL_OFFSET.getValue()); + keyBuf.writeByte(RecordPrefix.DATA.getValue()); + keyBuf.writeShort(groupBytes.length); + keyBuf.writeBytes(groupBytes); + keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1); + keyBuf.writeShort(topicBytes.length); + keyBuf.writeBytes(topicBytes); + keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1); + keyBuf.writeInt(queueId); + return keyBuf; + } + + @Override + public boolean load() { + return loadDataVersion() && loadConsumerOffsets(); + } + + @Override + public synchronized void persist() { + try { + configStorage.flushWAL(); + } catch (RocksDBException e) { + LOG.error("Failed to flush RocksDB config instance WAL", e); + } + } + + /** + *

    + * Layout of data version key: + * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes] + *

    + * + *

    + * Layout of data version value: + * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes] + *

    + */ + public boolean loadDataVersion() { + try { + ConfigHelper.loadDataVersion(configStorage, TableId.CONSUMER_OFFSET) + .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion)); + } catch (RocksDBException e) { + LOG.error("Failed to load RocksDB config", e); + return false; + } + return true; + } + + private boolean loadConsumerOffsets() { + // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte] + ByteBuf beginKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4); + beginKeyBuf.writeByte(TablePrefix.TABLE.getValue()); + beginKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue()); + beginKeyBuf.writeByte(RecordPrefix.DATA.getValue()); + + ByteBuf endKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4); + endKeyBuf.writeByte(TablePrefix.TABLE.getValue()); + endKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue()); + endKeyBuf.writeByte(RecordPrefix.DATA.getValue() + 1); + + try (RocksIterator iterator = configStorage.iterate(beginKeyBuf.nioBuffer(), endKeyBuf.nioBuffer())) { + int keyCapacity = 256; + // We may iterate millions of LMQ consumer offsets here, use direct byte buffers here to avoid memory + // fragment + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyCapacity); + ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES); + while (iterator.isValid()) { + keyBuffer.clear(); + valueBuffer.clear(); + + int len = iterator.key(keyBuffer); + if (len > keyCapacity) { + keyCapacity = len; + PlatformDependent.freeDirectBuffer(keyBuffer); + // Reserve more space for key + keyBuffer = ByteBuffer.allocateDirect(keyCapacity); + continue; + } + len = iterator.value(valueBuffer); + assert len == Long.BYTES; + + // skip table-prefix, table-id, record-prefix + keyBuffer.position(1 + 2 + 1); + short groupLen = keyBuffer.getShort(); + byte[] groupBytes = new byte[groupLen]; + keyBuffer.get(groupBytes); + byte ctrl = keyBuffer.get(); + assert ctrl == AbstractRocksDBStorage.CTRL_1; + + short topicLen = keyBuffer.getShort(); + byte[] topicBytes = new byte[topicLen]; + keyBuffer.get(topicBytes); + String topic = new String(topicBytes, StandardCharsets.UTF_8); + ctrl = keyBuffer.get(); + assert ctrl == AbstractRocksDBStorage.CTRL_1; + + int queueId = keyBuffer.getInt(); + + long offset = valueBuffer.getLong(); + + if (!MixAll.isLmq(topic)) { + String group = new String(groupBytes, StandardCharsets.UTF_8); + onConsumerOffsetRecordLoad(topic, group, queueId, offset); + } + iterator.next(); + } + PlatformDependent.freeDirectBuffer(keyBuffer); + PlatformDependent.freeDirectBuffer(valueBuffer); + } finally { + beginKeyBuf.release(); + endKeyBuf.release(); + } + return true; + } + + private void onConsumerOffsetRecordLoad(String topic, String group, int queueId, long offset) { + if (MixAll.isLmq(topic)) { + return; + } + String key = topic + TOPIC_GROUP_SEPARATOR + group; + if (!offsetTable.containsKey(key)) { + ConcurrentMap map = new ConcurrentHashMap<>(); + offsetTable.putIfAbsent(key, map); + } + offsetTable.get(key).put(queueId, offset); + } + + @Override + public long queryOffset(String group, String topic, int queueId) { + if (!MixAll.isLmq(topic)) { + return super.queryOffset(group, topic, queueId); + } + + ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId); + try { + byte[] slice = configStorage.get(keyBuf.nioBuffer()); + if (null == slice) { + return -1; + } + assert slice.length == Long.BYTES; + return ByteBuffer.wrap(slice).getLong(); + } catch (RocksDBException e) { + throw new RuntimeException(e); + } finally { + keyBuf.release(); + } + } + + @Override + public void commitPullOffset(String clientHost, String group, String topic, int queueId, long offset) { + if (!MixAll.isLmq(topic)) { + super.commitPullOffset(clientHost, group, topic, queueId, offset); + } + + ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId); + ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(8); + try (WriteBatch writeBatch = new WriteBatch()) { + writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + } catch (RocksDBException e) { + LOG.error("Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}", + group, topic, queueId, offset); + } finally { + keyBuf.release(); + valueBuf.release(); + } + } + + @Override + public long queryPullOffset(String group, String topic, int queueId) { + if (!MixAll.isLmq(topic)) { + return super.queryPullOffset(group, topic, queueId); + } + + ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId); + try { + byte[] valueBytes = configStorage.get(keyBuf.nioBuffer()); + if (null == valueBytes) { + return -1; + } + return ByteBuffer.wrap(valueBytes).getLong(); + } catch (RocksDBException e) { + LOG.error("Failed to queryPullOffset. group={}, topic={}, queueId={}", group, topic, queueId); + } finally { + keyBuf.release(); + } + return -1; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java new file mode 100644 index 00000000000..750d454d4ee --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +public enum RecordPrefix { + UNSPECIFIED((byte)0), + DATA_VERSION((byte)1), + DATA((byte)2); + + private final byte value; + + RecordPrefix(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java new file mode 100644 index 00000000000..2ee157fdc8d --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +public enum SerializationType { + UNSPECIFIED((byte) 0), + + JSON((byte) 1), + + PROTOBUF((byte) 2), + + FLAT_BUFFERS((byte) 3); + + private final byte value; + + SerializationType(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } + + public static SerializationType valueOf(byte value) { + for (SerializationType type : SerializationType.values()) { + if (type.getValue() == value) { + return type; + } + } + return SerializationType.UNSPECIFIED; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java new file mode 100644 index 00000000000..8da6f9d2bc5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +import com.alibaba.fastjson2.JSON; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.nio.charset.StandardCharsets; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; + +public class SubscriptionGroupManagerV2 extends SubscriptionGroupManager { + + private final ConfigStorage configStorage; + + public SubscriptionGroupManagerV2(BrokerController brokerController, ConfigStorage configStorage) { + super(brokerController); + this.configStorage = configStorage; + } + + @Override + public boolean load() { + return loadDataVersion() && loadSubscriptions(); + } + + public boolean loadDataVersion() { + try { + ConfigHelper.loadDataVersion(configStorage, TableId.SUBSCRIPTION_GROUP) + .ifPresent(buf -> { + ConfigHelper.onDataVersionLoad(buf, dataVersion); + }); + } catch (RocksDBException e) { + log.error("loadDataVersion error", e); + return false; + } + return true; + } + + private boolean loadSubscriptions() { + int keyLen = 1 /* table prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */; + ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + beginKey.writeByte(TablePrefix.TABLE.getValue()); + beginKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue()); + beginKey.writeByte(RecordPrefix.DATA.getValue()); + + ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + endKey.writeByte(TablePrefix.TABLE.getValue()); + endKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue()); + endKey.writeByte(RecordPrefix.DATA.getValue() + 1); + + try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) { + while (iterator.isValid()) { + SubscriptionGroupConfig subscriptionGroupConfig = parseSubscription(iterator.key(), iterator.value()); + if (null != subscriptionGroupConfig) { + super.updateSubscriptionGroupConfigWithoutPersist(subscriptionGroupConfig); + } + } + } finally { + beginKey.release(); + endKey.release(); + } + return true; + } + + private SubscriptionGroupConfig parseSubscription(byte[] key, byte[] value) { + ByteBuf keyBuf = Unpooled.wrappedBuffer(key); + ByteBuf valueBuf = Unpooled.wrappedBuffer(value); + try { + // Skip table-prefix, table-id, record-type-prefix + keyBuf.readerIndex(4); + short groupNameLen = keyBuf.readShort(); + assert groupNameLen == keyBuf.readableBytes(); + CharSequence groupName = keyBuf.readCharSequence(groupNameLen, StandardCharsets.UTF_8); + assert null != groupName; + byte serializationType = valueBuf.readByte(); + if (SerializationType.JSON == SerializationType.valueOf(serializationType)) { + CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8); + SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(json.toString(), SubscriptionGroupConfig.class); + assert subscriptionGroupConfig != null; + assert groupName.equals(subscriptionGroupConfig.getGroupName()); + return subscriptionGroupConfig; + } + } finally { + keyBuf.release(); + valueBuf.release(); + } + return null; + } + + @Override + public synchronized void persist() { + try { + configStorage.flushWAL(); + } catch (RocksDBException e) { + log.error("Failed to flush RocksDB WAL", e); + } + } + + @Override + public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) { + if (MixAll.isLmq(group)) { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(group); + return subscriptionGroupConfig; + } + return super.findSubscriptionGroupConfig(group); + } + + @Override + public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) { + if (config == null || MixAll.isLmq(config.getGroupName())) { + return; + } + ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, config.getGroupName()); + ByteBuf valueBuf = ConfigHelper.valueBufOf(config, SerializationType.JSON); + try (WriteBatch writeBatch = new WriteBatch()) { + writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + log.error("update subscription group config error", e); + } finally { + keyBuf.release(); + valueBuf.release(); + } + super.updateSubscriptionGroupConfigWithoutPersist(config); + } + + @Override + public boolean containsSubscriptionGroup(String group) { + if (MixAll.isLmq(group)) { + return true; + } else { + return super.containsSubscriptionGroup(group); + } + } + + @Override + protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) { + ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, groupName); + try (WriteBatch writeBatch = new WriteBatch()) { + writeBatch.delete(ConfigHelper.readBytes(keyBuf)); + long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion(); + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + } catch (RocksDBException e) { + log.error("Failed to remove subscription group config by group-name={}", groupName, e); + } + return super.removeSubscriptionGroupConfig(groupName); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java new file mode 100644 index 00000000000..7a61899371e --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +/** + * See Table, Key Value Mapping + */ +public enum TableId { + UNSPECIFIED((short) 0), + CONSUMER_OFFSET((short) 1), + PULL_OFFSET((short) 2), + TOPIC((short) 3), + SUBSCRIPTION_GROUP((short) 4); + + private final short value; + + TableId(short value) { + this.value = value; + } + + public short getValue() { + return value; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java new file mode 100644 index 00000000000..d16c14d275a --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +public enum TablePrefix { + UNSPECIFIED((byte) 0), + TABLE((byte) 1); + + private final byte value; + + TablePrefix(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java new file mode 100644 index 00000000000..b1a3d2d85ce --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +import com.alibaba.fastjson2.JSON; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.nio.charset.StandardCharsets; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.config.AbstractRocksDBStorage; +import org.apache.rocketmq.common.constant.PermName; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; + +/** + * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes] + * Value layout: [serialization-type, 1 byte][topic-config-bytes] + */ +public class TopicConfigManagerV2 extends TopicConfigManager { + private final ConfigStorage configStorage; + + public TopicConfigManagerV2(BrokerController brokerController, ConfigStorage configStorage) { + super(brokerController); + this.configStorage = configStorage; + } + + @Override + public boolean load() { + return loadDataVersion() && loadTopicConfig(); + } + + public boolean loadDataVersion() { + try { + ConfigHelper.loadDataVersion(configStorage, TableId.TOPIC) + .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion)); + } catch (RocksDBException e) { + log.error("Failed to load data version of topic", e); + return false; + } + return true; + } + + private boolean loadTopicConfig() { + int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */; + ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + beginKey.writeByte(TablePrefix.TABLE.getValue()); + beginKey.writeShort(TableId.TOPIC.getValue()); + beginKey.writeByte(RecordPrefix.DATA.getValue()); + + ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen); + endKey.writeByte(TablePrefix.TABLE.getValue()); + endKey.writeShort(TableId.TOPIC.getValue()); + endKey.writeByte(RecordPrefix.DATA.getValue() + 1); + + try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) { + while (iterator.isValid()) { + byte[] key = iterator.key(); + byte[] value = iterator.value(); + TopicConfig topicConfig = parseTopicConfig(key, value); + if (null != topicConfig) { + super.updateSingleTopicConfigWithoutPersist(topicConfig); + } + iterator.next(); + } + } finally { + beginKey.release(); + endKey.release(); + } + return true; + } + + /** + * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes] + * Value layout: [serialization-type, 1 byte][topic-config-bytes] + * + * @param key Topic config key representation in RocksDB + * @param value Topic config value representation in RocksDB + * @return decoded topic config + */ + private TopicConfig parseTopicConfig(byte[] key, byte[] value) { + ByteBuf keyBuf = Unpooled.wrappedBuffer(key); + ByteBuf valueBuf = Unpooled.wrappedBuffer(value); + try { + // Skip table-prefix, table-id, record-type-prefix + keyBuf.readerIndex(4); + short topicLen = keyBuf.readShort(); + assert topicLen == keyBuf.readableBytes(); + CharSequence topic = keyBuf.readCharSequence(topicLen, StandardCharsets.UTF_8); + assert null != topic; + + byte serializationType = valueBuf.readByte(); + if (SerializationType.JSON == SerializationType.valueOf(serializationType)) { + CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8); + TopicConfig topicConfig = JSON.parseObject(json.toString(), TopicConfig.class); + assert topicConfig != null; + assert topic.equals(topicConfig.getTopicName()); + return topicConfig; + } + } finally { + keyBuf.release(); + valueBuf.release(); + } + + return null; + } + + @Override + public synchronized void persist() { + try { + configStorage.flushWAL(); + } catch (RocksDBException e) { + log.error("Failed to flush WAL", e); + } + } + + @Override + public TopicConfig selectTopicConfig(final String topic) { + if (MixAll.isLmq(topic)) { + return simpleLmqTopicConfig(topic); + } + return super.selectTopicConfig(topic); + } + + @Override + public void updateTopicConfig(final TopicConfig topicConfig) { + if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) { + return; + } + super.updateSingleTopicConfigWithoutPersist(topicConfig); + + ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicConfig.getTopicName()); + ByteBuf valueBuf = ConfigHelper.valueBufOf(topicConfig, SerializationType.JSON); + try (WriteBatch writeBatch = new WriteBatch()) { + writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + log.error("Failed to update topic config", e); + } finally { + keyBuf.release(); + valueBuf.release(); + } + } + + @Override + protected TopicConfig removeTopicConfig(String topicName) { + ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicName); + try (WriteBatch writeBatch = new WriteBatch()) { + writeBatch.delete(keyBuf.nioBuffer()); + long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; + ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); + } catch (RocksDBException e) { + log.error("Failed to delete topic config by topicName={}", topicName, e); + } finally { + keyBuf.release(); + } + return super.removeTopicConfig(topicName); + } + + @Override + public boolean containsTopic(String topic) { + if (MixAll.isLmq(topic)) { + return true; + } + return super.containsTopic(topic); + } + + private TopicConfig simpleLmqTopicConfig(String topic) { + return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java new file mode 100644 index 00000000000..1ea216193c3 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.config.v2; + +/* + * Endian: we use network byte order for all integrals, aka, always big endian. + * + * Unlike v1 config managers, implementations in this package prioritize data integrity and reliability. + * As a result,RocksDB write-ahead-log is always on and changes are immediately flushed. Another significant + * difference is that heap-based cache is removed because it is not necessary and duplicated to RocksDB + * MemTable/BlockCache. + */ diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java index 403324137cc..ea46f1d8a1f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -43,7 +43,7 @@ public class ConsumerOffsetManager extends ConfigManager { protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); public static final String TOPIC_GROUP_SEPARATOR = "@"; - private DataVersion dataVersion = new DataVersion(); + protected DataVersion dataVersion = new DataVersion(); protected ConcurrentMap> offsetTable = new ConcurrentHashMap<>(512); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java index e6855ef9a2a..f62a3e4a091 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -49,7 +49,7 @@ public class SubscriptionGroupManager extends ConfigManager { private ConcurrentMap> forbiddenTable = new ConcurrentHashMap<>(4); - private final DataVersion dataVersion = new DataVersion(); + protected final DataVersion dataVersion = new DataVersion(); protected transient BrokerController brokerController; public SubscriptionGroupManager() { @@ -143,7 +143,7 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) this.persist(); } - private void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) { + protected void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) { Map newAttributes = request(config); Map currentAttributes = current(config.getGroupName()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 25d3218f2ab..4530c10002d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -66,7 +66,7 @@ public class TopicConfigManager extends ConfigManager { private transient final Lock topicConfigTableLock = new ReentrantLock(); protected ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(1024); - private DataVersion dataVersion = new DataVersion(); + protected DataVersion dataVersion = new DataVersion(); protected transient BrokerController brokerController; public TopicConfigManager() { @@ -497,7 +497,7 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) } } - private void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) { + protected void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) { checkNotNull(topicConfig, "topicConfig shouldn't be null"); Map newAttributes = request(topicConfig); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java index 58b690c9a34..5a7a2c38acb 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManagerTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.After; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java index ea6528546dc..1b9916d6ac1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.offset; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBLmqConsumerOffsetManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.store.config.MessageStoreConfig; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java index dde0401e8ae..c01e63f31f7 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBOffsetSerializeWrapperTest.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.broker.config.v1.RocksDBOffsetSerializeWrapper; import org.junit.Before; import org.junit.Test; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java index b4800aec24e..64c505eb77c 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.commons.collections.MapUtils; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index 04324043fb8..d87f5133552 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -36,8 +36,8 @@ import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.broker.schedule.ScheduleMessageService; -import org.apache.rocketmq.broker.subscription.RocksDBSubscriptionGroupManager; -import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager; +import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager; +import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.BrokerConfig; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java index 205e642843b..26017af8a64 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.subscription; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.remoting.protocol.DataVersion; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java index b0e0d057363..080e1dd5a39 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicAttributes; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java index 2a727090987..fb345548e4f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.topic; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 2acfdd69a5c..c6510476617 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common; import org.apache.rocketmq.common.annotation.ImportantField; +import org.apache.rocketmq.common.config.ConfigManagerVersion; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageRequestMode; import org.apache.rocketmq.common.metrics.MetricsExporterType; @@ -431,6 +432,11 @@ public class BrokerConfig extends BrokerIdentity { private boolean appendCkAsync = false; + /** + * V2 is recommended in cases where LMQ feature is extensively used. + */ + private String configManagerVersion = ConfigManagerVersion.V1.getVersion(); + public String getConfigBlackList() { return configBlackList; } @@ -1879,4 +1885,12 @@ public boolean isAppendCkAsync() { public void setAppendCkAsync(boolean appendCkAsync) { this.appendCkAsync = appendCkAsync; } + + public String getConfigManagerVersion() { + return configManagerVersion; + } + + public void setConfigManagerVersion(String configManagerVersion) { + this.configManagerVersion = configManagerVersion; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java new file mode 100644 index 00000000000..0d5dd6940a1 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigManagerVersion.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.config; + +public enum ConfigManagerVersion { + V1("v1"), + V2("v2"), + ; + private final String version; + + ConfigManagerVersion(String version) { + this.version = version; + } + + public String getVersion() { + return version; + } +} From ecb45bb90dba46bb35b51520b01890c9c47ba55c Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 28 Oct 2024 16:32:58 +0800 Subject: [PATCH 202/265] [ISSUE #8852] Add more test coverage for ClientMetadata (#8853) * [ISSUE #8852] Add more test coverage for ClientMetadata * Update * Update * Update --- .../remoting/rpc/ClientMetadataTest.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java new file mode 100644 index 00000000000..a9f38854586 --- /dev/null +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/ClientMetadataTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.rpc; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class ClientMetadataTest { + + private ClientMetadata clientMetadata; + + private final ConcurrentMap topicRouteTable = new ConcurrentHashMap<>(); + + private final ConcurrentMap> topicEndPointsTable = new ConcurrentHashMap<>(); + + private final ConcurrentMap> brokerAddrTable = new ConcurrentHashMap<>(); + + private final String defaultTopic = "defaultTopic"; + + private final String defaultBroker = "defaultBroker"; + + @Before + public void init() throws IllegalAccessException { + clientMetadata = new ClientMetadata(); + + FieldUtils.writeDeclaredField(clientMetadata, "topicRouteTable", topicRouteTable, true); + FieldUtils.writeDeclaredField(clientMetadata, "topicEndPointsTable", topicEndPointsTable, true); + FieldUtils.writeDeclaredField(clientMetadata, "brokerAddrTable", brokerAddrTable, true); + } + + @Test + public void testGetBrokerNameFromMessageQueue() { + MessageQueue mq1 = new MessageQueue(defaultTopic, "broker0", 0); + MessageQueue mq2 = new MessageQueue(defaultTopic, "broker1", 0); + ConcurrentMap messageQueueMap = new ConcurrentHashMap<>(); + messageQueueMap.put(mq1, "broker0"); + messageQueueMap.put(mq2, "broker1"); + topicEndPointsTable.put(defaultTopic, messageQueueMap); + + String actual = clientMetadata.getBrokerNameFromMessageQueue(mq1); + assertEquals("broker0", actual); + } + + @Test + public void testGetBrokerNameFromMessageQueueNotFound() { + MessageQueue mq = new MessageQueue("topic1", "broker0", 0); + topicEndPointsTable.put(defaultTopic, new ConcurrentHashMap<>()); + + String actual = clientMetadata.getBrokerNameFromMessageQueue(mq); + assertEquals("broker0", actual); + } + + @Test + public void testFindMasterBrokerAddrNotFound() { + assertNull(clientMetadata.findMasterBrokerAddr(defaultBroker)); + } + + @Test + public void testFindMasterBrokerAddr() { + String defaultBrokerAddr = "127.0.0.1:10911"; + brokerAddrTable.put(defaultBroker, new HashMap<>()); + brokerAddrTable.get(defaultBroker).put(0L, defaultBrokerAddr); + + String actual = clientMetadata.findMasterBrokerAddr(defaultBroker); + assertEquals(defaultBrokerAddr, actual); + } + + @Test + public void testTopicRouteData2EndpointsForStaticTopicNotFound() { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setTopicQueueMappingByBroker(null); + + ConcurrentMap actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData); + assertTrue(actual.isEmpty()); + } + + @Test + public void testTopicRouteData2EndpointsForStaticTopic() { + TopicRouteData topicRouteData = new TopicRouteData(); + Map mappingInfos = new HashMap<>(); + TopicQueueMappingInfo info = new TopicQueueMappingInfo(); + info.setScope("scope"); + info.setCurrIdMap(new ConcurrentHashMap<>()); + info.getCurrIdMap().put(0, 0); + info.setTotalQueues(1); + info.setBname("bname"); + mappingInfos.put(defaultBroker, info); + topicRouteData.setTopicQueueMappingByBroker(mappingInfos); + + ConcurrentMap actual = ClientMetadata.topicRouteData2EndpointsForStaticTopic(defaultTopic, topicRouteData); + assertEquals(1, actual.size()); + } +} From a96a14f734fb4e3260c4a5285f400e1fb2ec513c Mon Sep 17 00:00:00 2001 From: zhaohai <33314633+zhaohai666@users.noreply.github.com> Date: Tue, 29 Oct 2024 19:07:14 +0800 Subject: [PATCH 203/265] update LanguageCode (#8872) Co-authored-by: zh378814 --- .../org/apache/rocketmq/remoting/protocol/LanguageCode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java index 2df9fbf0278..cf43cbab3dd 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java @@ -35,7 +35,8 @@ public enum LanguageCode { GO((byte) 9), PHP((byte) 10), OMS((byte) 11), - RUST((byte) 12); + RUST((byte) 12), + NODE_JS((byte) 13); private byte code; From 0b247685007258d1502fa992434402fd40b92cea Mon Sep 17 00:00:00 2001 From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> Date: Tue, 29 Oct 2024 19:09:35 +0800 Subject: [PATCH 204/265] [ISSUE #8822] Double write cq, reduce unnecessary switches (#8823) * Reduce unnecessary switches --- .../rocketmq/broker/RocksDBConfigManager.java | 14 +- .../v1/RocksDBConsumerOffsetManager.java | 9 +- .../v1/RocksDBSubscriptionGroupManager.java | 8 +- .../config/v1/RocksDBTopicConfigManager.java | 8 +- .../processor/AdminBrokerProcessor.java | 198 ++++++++++++------ .../RocksdbTransferOffsetAndCqTest.java | 1 - .../RocksdbGroupConfigTransferTest.java | 1 - .../topic/RocksdbTopicConfigManagerTest.java | 1 - .../topic/RocksdbTopicConfigTransferTest.java | 1 - .../rocketmq/client/impl/MQClientAPIImpl.java | 7 +- .../common/CheckRocksdbCqWriteResult.java | 38 +++- .../common/config/AbstractRocksDBStorage.java | 3 +- .../common/config/ConfigRocksDBStorage.java | 12 +- ...ckRocksdbCqWriteProgressRequestHeader.java | 11 + .../rocketmq/store/RocksDBMessageStore.java | 4 + .../store/config/MessageStoreConfig.java | 32 ++- .../store/rocksdb/RocksDBOptionsFactory.java | 5 +- .../tools/admin/DefaultMQAdminExt.java | 6 +- .../tools/admin/DefaultMQAdminExtImpl.java | 6 +- .../rocketmq/tools/admin/MQAdminExt.java | 5 +- .../CheckRocksdbCqWriteProgressCommand.java | 21 +- 21 files changed, 246 insertions(+), 145 deletions(-) rename remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java => common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java (52%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java index 20358c4707f..ee2d4e54a6a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java @@ -17,40 +17,40 @@ package org.apache.rocketmq.broker; import com.alibaba.fastjson.JSON; +import java.nio.charset.StandardCharsets; +import java.util.function.BiConsumer; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.config.ConfigRocksDBStorage; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.rocksdb.CompressionType; import org.rocksdb.FlushOptions; import org.rocksdb.RocksIterator; import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; -import java.nio.charset.StandardCharsets; -import java.util.function.BiConsumer; - public class RocksDBConfigManager { protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); public volatile boolean isStop = false; public ConfigRocksDBStorage configRocksDBStorage = null; private FlushOptions flushOptions = null; private volatile long lastFlushMemTableMicroSecond = 0; - private final String filePath; private final long memTableFlushInterval; + private final CompressionType compressionType; private DataVersion kvDataVersion = new DataVersion(); - - public RocksDBConfigManager(String filePath, long memTableFlushInterval) { + public RocksDBConfigManager(String filePath, long memTableFlushInterval, CompressionType compressionType) { this.filePath = filePath; this.memTableFlushInterval = memTableFlushInterval; + this.compressionType = compressionType; } public boolean init() { this.isStop = false; - this.configRocksDBStorage = new ConfigRocksDBStorage(filePath); + this.configRocksDBStorage = new ConfigRocksDBStorage(filePath, compressionType); return this.configRocksDBStorage.start(); } public boolean loadDataVersion() { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java index 8066fe769a0..824fc0fee3e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBConsumerOffsetManager.java @@ -31,6 +31,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.rocksdb.CompressionType; import org.rocksdb.WriteBatch; public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { @@ -41,7 +42,9 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { public RocksDBConsumerOffsetManager(BrokerController brokerController) { super(brokerController); - this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(), + CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType())); + } @Override @@ -61,10 +64,6 @@ public boolean loadConsumerOffset() { } private boolean merge() { - if (!brokerController.getMessageStoreConfig().isTransferOffsetJsonToRocksdb()) { - log.info("the switch transferOffsetJsonToRocksdb is off, no merge offset operation is needed."); - return true; - } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { log.info("consumerOffset json file does not exist, so skip merge"); return true; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java index 8175d63cce1..8fc7a4d6edb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBSubscriptionGroupManager.java @@ -32,6 +32,7 @@ import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.rocksdb.CompressionType; import org.rocksdb.RocksIterator; public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { @@ -40,7 +41,8 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { public RocksDBSubscriptionGroupManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(), + CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType())); } @Override @@ -78,10 +80,6 @@ public boolean loadForbidden(BiConsumer biConsumer) { private boolean merge() { - if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("the switch transferMetadataJsonToRocksdb is off, no merge subGroup operation is needed."); - return true; - } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { log.info("subGroup json file does not exist, so skip merge"); return true; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java index bce67392f64..18e633d348b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBTopicConfigManager.java @@ -28,6 +28,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.remoting.protocol.DataVersion; +import org.rocksdb.CompressionType; public class RocksDBTopicConfigManager extends TopicConfigManager { @@ -35,7 +36,8 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { public RocksDBTopicConfigManager(BrokerController brokerController) { super(brokerController, false); - this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs(), + CompressionType.getCompressionType(brokerController.getMessageStoreConfig().getRocksdbCompressionType())); } @Override @@ -59,10 +61,6 @@ public boolean loadDataVersion() { } private boolean merge() { - if (!brokerController.getMessageStoreConfig().isTransferMetadataJsonToRocksdb()) { - log.info("the switch transferMetadataJsonToRocksdb is off, no merge topic operation is needed."); - return true; - } if (!UtilAll.isPathExists(this.configFilePath()) && !UtilAll.isPathExists(this.configFilePath() + ".bak")) { log.info("topic json file does not exist, so skip merge"); return true; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index aa962513df3..381889c6247 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -18,7 +18,6 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; @@ -39,6 +38,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -65,7 +67,9 @@ import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; +import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.LockCallback; import org.apache.rocketmq.common.MQVersion; @@ -214,6 +218,7 @@ import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.StoreType; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore; @@ -232,6 +237,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); protected final BrokerController brokerController; protected Set configBlackList = new HashSet<>(); + private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new SynchronousQueue<>()); public AdminBrokerProcessor(final BrokerController brokerController) { this.brokerController = brokerController; @@ -467,76 +473,23 @@ private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, Re return response; } - private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { - CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class); - String requestTopic = requestHeader.getTopic(); - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - MessageStore messageStore = brokerController.getMessageStore(); - DefaultMessageStore defaultMessageStore; - if (messageStore instanceof AbstractPluginMessageStore) { - defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext(); - } else { - defaultMessageStore = (DefaultMessageStore) messageStore; - } - RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore(); - if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { - response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", "rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid"))); - return response; - } - - ConcurrentMap> cqTable = defaultMessageStore.getConsumeQueueTable(); - StringBuilder diffResult = new StringBuilder(); - try { - if (StringUtils.isNotBlank(requestTopic)) { - processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult,false); - response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); - return response; - } - for (Map.Entry> topicEntry : cqTable.entrySet()) { - String topic = topicEntry.getKey(); - processConsumeQueuesForTopic(topicEntry.getValue(), topic, rocksDBMessageStore, diffResult,true); + private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) { + CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult(); + result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_IN_PROGRESS.getValue()); + Runnable runnable = () -> { + try { + CheckRocksdbCqWriteResult checkResult = doCheckRocksdbCqWriteProgress(ctx, request); + LOGGER.info("checkRocksdbCqWriteProgress result: {}", JSON.toJSONString(checkResult)); + } catch (Exception e) { + LOGGER.error("checkRocksdbCqWriteProgress error", e); } - diffResult.append("check all topic successful, size:").append(cqTable.size()); - response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", diffResult.toString()))); - - } catch (Exception e) { - LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e); - response.setBody(JSON.toJSONBytes(ImmutableMap.of("diffResult", e.getMessage()))); - } + }; + asyncExecuteWorker.submit(runnable); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(JSON.toJSONBytes(result)); return response; } - - private void processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean checkAll) { - for (Map.Entry queueEntry : queueMap.entrySet()) { - Integer queueId = queueEntry.getKey(); - ConsumeQueueInterface jsonCq = queueEntry.getValue(); - ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId); - if (!checkAll) { - String format = String.format("\n[topic: %s, queue: %s] \n kvEarliest : %s | kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ", - topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit()); - diffResult.append(format).append("\n"); - } - long maxFileOffsetInQueue = jsonCq.getMaxOffsetInQueue(); - long minOffsetInQueue = kvCq.getMinOffsetInQueue(); - for (long i = minOffsetInQueue; i < maxFileOffsetInQueue; i++) { - Pair fileCqUnit = jsonCq.getCqUnitAndStoreTime(i); - Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i); - if (fileCqUnit == null || kvCqUnit == null) { - diffResult.append(String.format("[topic: %s, queue: %s, offset: %s] \n kv : %s \n file : %s \n", - topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null")); - return; - } - if (!checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) { - String diffInfo = String.format("[topic:%s, queue: %s offset: %s] \n file : %s \n kv : %s \n", - topic, queueId, i, kvCqUnit.getObject1(), fileCqUnit.getObject1()); - LOGGER.error(diffInfo); - diffResult.append(diffInfo).append(System.lineSeparator()); - return; - } - } - } - } @Override public boolean rejectRequest() { return false; @@ -3418,6 +3371,115 @@ private boolean validateBlackListConfigExist(Properties properties) { return false; } + private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class); + String requestTopic = requestHeader.getTopic(); + MessageStore messageStore = brokerController.getMessageStore(); + DefaultMessageStore defaultMessageStore; + if (messageStore instanceof AbstractPluginMessageStore) { + defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext(); + } else { + defaultMessageStore = (DefaultMessageStore) messageStore; + } + RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore(); + CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult(); + + if (defaultMessageStore.getMessageStoreConfig().getStoreType().equals(StoreType.DEFAULT_ROCKSDB.getStoreType())) { + result.setCheckResult("storeType is DEFAULT_ROCKSDB, no need check"); + result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue()); + return result; + } + + if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) { + result.setCheckResult("rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid"); + result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue()); + return result; + } + + ConcurrentMap> cqTable = defaultMessageStore.getConsumeQueueTable(); + StringBuilder diffResult = new StringBuilder(); + try { + if (StringUtils.isNotBlank(requestTopic)) { + boolean checkResult = processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult, true, requestHeader.getCheckStoreTime()); + result.setCheckResult(diffResult.toString()); + result.setCheckStatus(checkResult ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue()); + return result; + } + int successNum = 0; + int checkSize = 0; + for (Map.Entry> topicEntry : cqTable.entrySet()) { + boolean checkResult = processConsumeQueuesForTopic(topicEntry.getValue(), topicEntry.getKey(), rocksDBMessageStore, diffResult, false, requestHeader.getCheckStoreTime()); + successNum += checkResult ? 1 : 0; + checkSize++; + } + // check all topic finish, all topic is ready, checkSize: 100, currentQueueNum: 110 -> ready (The currentQueueNum means when we do checking, new topics are added.) + // check all topic finish, success/all : 89/100, currentQueueNum: 110 -> not ready + boolean checkReady = successNum == checkSize; + String checkResultString = checkReady ? String.format("all topic is ready, checkSize: %s, currentQueueNum: %s", checkSize, cqTable.size()) : + String.format("success/all : %s/%s, currentQueueNum: %s", successNum, checkSize, cqTable.size()); + diffResult.append("check all topic finish, ").append(checkResultString); + result.setCheckResult(diffResult.toString()); + result.setCheckStatus(checkReady ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue()); + } catch (Exception e) { + LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e); + result.setCheckResult(e.getMessage() + Arrays.toString(e.getStackTrace())); + result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue()); + } + return result; + } + + private boolean processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean printDetail, long checkpointByStoreTime) { + boolean processResult = true; + for (Map.Entry queueEntry : queueMap.entrySet()) { + Integer queueId = queueEntry.getKey(); + ConsumeQueueInterface jsonCq = queueEntry.getValue(); + ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId); + if (printDetail) { + String format = String.format("[topic: %s, queue: %s] \n kvEarliest : %s | kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ", + topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit()); + diffResult.append(format).append("\n"); + } + + long minOffsetByTime = 0L; + try { + minOffsetByTime = rocksDBMessageStore.getConsumeQueueStore().getOffsetInQueueByTime(topic, queueId, checkpointByStoreTime, BoundaryType.UPPER); + } catch (Exception e) { + // ignore + } + long minOffsetInQueue = kvCq.getMinOffsetInQueue(); + long checkFrom = Math.max(minOffsetInQueue, minOffsetByTime); + long checkTo = jsonCq.getMaxOffsetInQueue() - 1; + /* + checkTo(maxOffsetInQueue - 1) + v + fileCq +------------------------------------------------------+ + kvCq +----------------------------------------------+ + ^ ^ + minOffsetInQueue minOffsetByTime + ^ + checkFrom = max(minOffsetInQueue, minOffsetByTime) + */ + // The latest message is earlier than the check time + Pair fileLatestCq = jsonCq.getCqUnitAndStoreTime(checkTo); + if (fileLatestCq != null) { + if (fileLatestCq.getObject2() < checkpointByStoreTime) { + continue; + } + } + for (long i = checkFrom; i <= checkTo; i++) { + Pair fileCqUnit = jsonCq.getCqUnitAndStoreTime(i); + Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i); + if (fileCqUnit == null || kvCqUnit == null || !checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) { + LOGGER.error(String.format("[topic: %s, queue: %s, offset: %s] \n file : %s \n kv : %s \n", + topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null")); + processResult = false; + break; + } + } + } + return processResult; + } + private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) { if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) { return false; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java index 64c505eb77c..4b320eb53f3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java @@ -76,7 +76,6 @@ public void init() throws IOException { brokerConfig.setConsumerOffsetUpdateVersionStep(10); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); messageStoreConfig.setStorePathRootDir(basePath); - messageStoreConfig.setTransferOffsetJsonToRocksdb(true); messageStoreConfig.setRocksdbCQDoubleWriteEnable(true); Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java index 26017af8a64..c75fe0d6a03 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/subscription/RocksdbGroupConfigTransferTest.java @@ -68,7 +68,6 @@ public void init() { Mockito.lenient().when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); messageStoreConfig.setStorePathRootDir(basePath); - messageStoreConfig.setTransferMetadataJsonToRocksdb(true); Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java index 080e1dd5a39..fa3ef95f55f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigManagerTest.java @@ -72,7 +72,6 @@ public void init() { when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); messageStoreConfig.setStorePathRootDir(basePath); - messageStoreConfig.setTransferMetadataJsonToRocksdb(true); when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); Mockito.lenient().when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); Mockito.lenient().when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java index fb345548e4f..e925ed4bd8a 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/RocksdbTopicConfigTransferTest.java @@ -69,7 +69,6 @@ public void init() { when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); messageStoreConfig.setStorePathRootDir(basePath); - messageStoreConfig.setTransferMetadataJsonToRocksdb(true); Mockito.lenient().when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); when(brokerController.getMessageStore()).thenReturn(defaultMessageStore); when(defaultMessageStore.getStateMachineVersion()).thenReturn(0L); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 716d081ef46..554b1efa524 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -56,6 +56,7 @@ import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.client.rpchook.NamespaceRpcHook; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; @@ -113,7 +114,6 @@ import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; -import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; @@ -3019,15 +3019,16 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(final String brokerAddr, throw new MQClientException(response.getCode(), response.getRemark()); } - public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long timeoutMillis) throws InterruptedException, + public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(final String brokerAddr, final String topic, final long checkStoreTime, final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { CheckRocksdbCqWriteProgressRequestHeader header = new CheckRocksdbCqWriteProgressRequestHeader(); header.setTopic(topic); + header.setCheckStoreTime(checkStoreTime); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS, header); RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); assert response != null; if (ResponseCode.SUCCESS == response.getCode()) { - return CheckRocksdbCqWriteProgressResponseBody.decode(response.getBody(), CheckRocksdbCqWriteProgressResponseBody.class); + return JSON.parseObject(response.getBody(), CheckRocksdbCqWriteResult.class); } throw new MQClientException(response.getCode(), response.getRemark()); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java b/common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java similarity index 52% rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java rename to common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java index 76719ac1a24..fc67df86c2f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/CheckRocksdbCqWriteProgressResponseBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/CheckRocksdbCqWriteResult.java @@ -15,21 +15,43 @@ * limitations under the License. */ -package org.apache.rocketmq.remoting.protocol.body; +package org.apache.rocketmq.common; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +public class CheckRocksdbCqWriteResult { + String checkResult; -public class CheckRocksdbCqWriteProgressResponseBody extends RemotingSerializable { + int checkStatus; - String diffResult; + public enum CheckStatus { + CHECK_OK(0), + CHECK_NOT_OK(1), + CHECK_IN_PROGRESS(2), + CHECK_ERROR(3); - public String getDiffResult() { - return diffResult; + private int value; + + CheckStatus(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } + + public String getCheckResult() { + return checkResult; } - public void setDiffResult(String diffResult) { - this.diffResult = diffResult; + public void setCheckResult(String checkResult) { + this.checkResult = checkResult; } + public int getCheckStatus() { + return checkStatus; + } + public void setCheckStatus(int checkStatus) { + this.checkStatus = checkStatus; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index 42ddbdc728c..d434cce7451 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -86,6 +86,7 @@ public abstract class AbstractRocksDBStorage { protected final List cfHandles = new ArrayList<>(); protected volatile boolean loaded; + protected CompressionType compressionType = CompressionType.LZ4_COMPRESSION; private volatile boolean closed; private final Semaphore reloadPermit = new Semaphore(1); @@ -156,7 +157,7 @@ protected void initCompactRangeOptions() { protected void initCompactionOptions() { this.compactionOptions = new CompactionOptions(); - this.compactionOptions.setCompression(CompressionType.LZ4_COMPRESSION); + this.compactionOptions.setCompression(compressionType); this.compactionOptions.setMaxSubcompactions(4); this.compactionOptions.setOutputFileSizeLimit(4 * 1024 * 1024 * 1024L); } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index 36da6834ff3..3b924a6a0d2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -25,6 +25,7 @@ import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.CompressionType; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -37,12 +38,17 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { protected ColumnFamilyHandle kvDataVersionFamilyHandle; protected ColumnFamilyHandle forbiddenFamilyHandle; - public static final byte[] KV_DATA_VERSION_KEY = "kvDataVersionKey".getBytes(StandardCharsets.UTF_8); + + public ConfigRocksDBStorage(final String dbPath) { - super(dbPath); - this.readOnly = false; + this(dbPath, false); + } + + public ConfigRocksDBStorage(final String dbPath, CompressionType compressionType) { + this(dbPath, false); + this.compressionType = compressionType; } public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java index fee158b4976..f679077fdde 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/CheckRocksdbCqWriteProgressRequestHeader.java @@ -32,6 +32,8 @@ public class CheckRocksdbCqWriteProgressRequestHeader implements CommandCustomHe @RocketMQResource(ResourceType.TOPIC) private String topic; + private long checkStoreTime; + @Override public void checkFields() throws RemotingCommandException { @@ -44,4 +46,13 @@ public String getTopic() { public void setTopic(String topic) { this.topic = topic; } + + public long getCheckStoreTime() { + return checkStoreTime; + } + + public void setCheckStoreTime(long checkStoreTime) { + this.checkStoreTime = checkStoreTime; + } + } diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 0a7119cab1d..321689ac8f5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -173,6 +173,10 @@ public CommitLogDispatcherBuildRocksdbConsumeQueue getDispatcherBuildRocksdbCons class CommitLogDispatcherBuildRocksdbConsumeQueue implements CommitLogDispatcher { @Override public void dispatch(DispatchRequest request) throws RocksDBException { + boolean enable = getMessageStoreConfig().isRocksdbCQDoubleWriteEnable(); + if (!enable) { + return; + } final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); switch (tranType) { case MessageSysFlag.TRANSACTION_NOT_TYPE: diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index e31c03dd22b..fe090e3fa2a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -22,6 +22,7 @@ import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.StoreType; import org.apache.rocketmq.store.queue.BatchConsumeQueue; +import org.rocksdb.CompressionType; public class MessageStoreConfig { @@ -106,8 +107,6 @@ public class MessageStoreConfig { @ImportantField private String storeType = StoreType.DEFAULT.getStoreType(); - private boolean transferMetadataJsonToRocksdb = false; - // ConsumeQueue file size,default is 30W private int mappedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE; // enable consume queue ext @@ -424,8 +423,6 @@ public class MessageStoreConfig { private boolean putConsumeQueueDataByFileChannel = true; - private boolean transferOffsetJsonToRocksdb = false; - private boolean rocksdbCQDoubleWriteEnable = false; /** @@ -443,7 +440,17 @@ public class MessageStoreConfig { * * LZ4 is the recommended one. */ - private String bottomMostCompressionTypeForConsumeQueueStore = "zstd"; + private String bottomMostCompressionTypeForConsumeQueueStore = CompressionType.ZSTD_COMPRESSION.getLibraryName(); + + private String rocksdbCompressionType = CompressionType.LZ4_COMPRESSION.getLibraryName(); + + public String getRocksdbCompressionType() { + return rocksdbCompressionType; + } + + public void setRocksdbCompressionType(String compressionType) { + this.rocksdbCompressionType = compressionType; + } /** * Spin number in the retreat strategy of spin lock @@ -464,13 +471,6 @@ public void setRocksdbCQDoubleWriteEnable(boolean rocksdbWriteEnable) { this.rocksdbCQDoubleWriteEnable = rocksdbWriteEnable; } - public boolean isTransferOffsetJsonToRocksdb() { - return transferOffsetJsonToRocksdb; - } - - public void setTransferOffsetJsonToRocksdb(boolean transferOffsetJsonToRocksdb) { - this.transferOffsetJsonToRocksdb = transferOffsetJsonToRocksdb; - } public boolean isEnabledAppendPropCRC() { return enabledAppendPropCRC; @@ -1894,14 +1894,6 @@ public void setPutConsumeQueueDataByFileChannel(boolean putConsumeQueueDataByFil this.putConsumeQueueDataByFileChannel = putConsumeQueueDataByFileChannel; } - public boolean isTransferMetadataJsonToRocksdb() { - return transferMetadataJsonToRocksdb; - } - - public void setTransferMetadataJsonToRocksdb(boolean transferMetadataJsonToRocksdb) { - this.transferMetadataJsonToRocksdb = transferMetadataJsonToRocksdb; - } - public String getBottomMostCompressionTypeForConsumeQueueStore() { return bottomMostCompressionTypeForConsumeQueueStore; } diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index d373ba6249c..66f5cbd095d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -67,13 +67,16 @@ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageSt setCompressionSizePercent(-1); String bottomMostCompressionTypeOpt = messageStore.getMessageStoreConfig() .getBottomMostCompressionTypeForConsumeQueueStore(); + String compressionTypeOpt = messageStore.getMessageStoreConfig() + .getRocksdbCompressionType(); CompressionType bottomMostCompressionType = CompressionType.getCompressionType(bottomMostCompressionTypeOpt); + CompressionType compressionType = CompressionType.getCompressionType(compressionTypeOpt); return columnFamilyOptions.setMaxWriteBufferNumber(4). setWriteBufferSize(128 * SizeUnit.MB). setMinWriteBufferNumberToMerge(1). setTableFormatConfig(blockBasedTableConfig). setMemTableConfig(new SkipListMemTableConfig()). - setCompressionType(CompressionType.LZ4_COMPRESSION). + setCompressionType(compressionType). setBottommostCompressionType(bottomMostCompressionType). setNumLevels(7). setCompactionStyle(CompactionStyle.UNIVERSAL). diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 3686bf2644b..c5ecdefb529 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; @@ -52,7 +53,6 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; -import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -773,9 +773,9 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String } @Override - public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { - return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic); + return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime); } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 883dcbe41d7..17f14f23af8 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -46,6 +46,7 @@ import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.common.KeyBuilder; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; @@ -90,7 +91,6 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; -import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -1819,9 +1819,9 @@ public QueryConsumeQueueResponseBody queryConsumeQueue(String brokerAddr, String } @Override - public CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) + public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { - return this.mqClientInstance.getMQClientAPIImpl().checkRocksdbCqWriteProgress(brokerAddr, topic, timeoutMillis); + return this.mqClientInstance.getMQClientAPIImpl().checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime, timeoutMillis); } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 09204ab7be2..aea43376eac 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.MQAdmin; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; @@ -48,7 +49,6 @@ import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList; import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; -import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; import org.apache.rocketmq.remoting.protocol.body.GroupList; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; @@ -149,7 +149,8 @@ ConsumeStats examineConsumeStats( final String consumerGroup) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; - CheckRocksdbCqWriteProgressResponseBody checkRocksdbCqWriteProgress(String brokerAddr, String topic) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; + CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr, String topic, long checkStoreTime) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; ConsumeStats examineConsumeStats(final String consumerGroup, final String topic) throws RemotingException, MQClientException, diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java index d18a24ee1dc..a0fc9fce1fb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/queue/CheckRocksdbCqWriteProgressCommand.java @@ -19,12 +19,13 @@ import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.CheckRocksdbCqWriteResult; import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.protocol.body.CheckRocksdbCqWriteProgressResponseBody; import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; @@ -53,8 +54,11 @@ public Options buildCommandlineOptions(Options options) { options.addOption(opt); opt = new Option("t", "topic", true, "topic name"); - opt.setRequired(false); options.addOption(opt); + + opt = new Option("cf", "checkFrom", true, "check from time"); + options.addOption(opt); + return options; } @@ -66,6 +70,10 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { defaultMQAdminExt.setNamesrvAddr(StringUtils.trim(commandLine.getOptionValue('n'))); String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : ""; String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : ""; + // The default check is 30 days + long checkStoreTime = commandLine.hasOption("cf") + ? Long.parseLong(commandLine.getOptionValue("cf").trim()) + : System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30L); try { defaultMQAdminExt.start(); @@ -80,14 +88,13 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { String brokerName = entry.getKey(); BrokerData brokerData = entry.getValue(); String brokerAddr = brokerData.getBrokerAddrs().get(0L); - CheckRocksdbCqWriteProgressResponseBody body = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic); - if (StringUtils.isNotBlank(topic)) { - System.out.print(body.getDiffResult()); + CheckRocksdbCqWriteResult result = defaultMQAdminExt.checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime); + if (result.getCheckStatus() == CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue()) { + System.out.print(brokerName + " check error, please check log... errInfo: " + result.getCheckResult()); } else { - System.out.print(brokerName + " | " + brokerAddr + " | \n" + body.getDiffResult()); + System.out.print(brokerName + " check doing, please wait and get the result from log... \n"); } } - } catch (Exception e) { throw new RuntimeException(this.getClass().getSimpleName() + " command failed", e); } finally { From 327abe5d6eb0b74f48867821447e2f0718509a42 Mon Sep 17 00:00:00 2001 From: zzjcool <92666291+zzjcool@users.noreply.github.com> Date: Tue, 29 Oct 2024 19:44:20 +0800 Subject: [PATCH 205/265] fix RequestCode overflowed issues:8868 (#8871) Co-authored-by: zhijiezheng --- .../remoting/protocol/RequestCode.java | 14 +++---- .../protocol/RocketMQSerializableTest.java | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index cfc5cc22785..1e10c96f428 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -88,13 +88,13 @@ public class RequestCode { public static final int GET_TIMER_METRICS = 61; - public static final int POP_MESSAGE = 200050; - public static final int ACK_MESSAGE = 200051; - public static final int BATCH_ACK_MESSAGE = 200151; - public static final int PEEK_MESSAGE = 200052; - public static final int CHANGE_MESSAGE_INVISIBLETIME = 200053; - public static final int NOTIFICATION = 200054; - public static final int POLLING_INFO = 200055; + public static final short POP_MESSAGE = 3442; + public static final short ACK_MESSAGE = 3443; + public static final short BATCH_ACK_MESSAGE = 3543; + public static final short PEEK_MESSAGE = 3444; + public static final short CHANGE_MESSAGE_INVISIBLETIME = 3445; + public static final short NOTIFICATION = 3446; + public static final short POLLING_INFO = 3447; public static final int PUT_KV_CONFIG = 100; diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java index 7cf32d70c34..c7533bd0b76 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java @@ -214,4 +214,41 @@ public void testFastEncode() throws Exception { assertThat(h2.getStr()).isEqualTo("s1"); assertThat(h2.getNum()).isEqualTo(100); } + + @Test + public void testRequestCodeEncode() throws Exception { + testRequestCodeEncode(RequestCode.POP_MESSAGE); + testRequestCodeEncode(RequestCode.ACK_MESSAGE); + testRequestCodeEncode(RequestCode.BATCH_ACK_MESSAGE); + testRequestCodeEncode(RequestCode.PEEK_MESSAGE); + testRequestCodeEncode(RequestCode.CHANGE_MESSAGE_INVISIBLETIME); + testRequestCodeEncode(RequestCode.NOTIFICATION); + testRequestCodeEncode(RequestCode.POLLING_INFO); + } + + public void testRequestCodeEncode(int code) throws Exception { + ByteBuf buf = ByteBufAllocator.DEFAULT.buffer(16); + MyHeader1 header1 = new MyHeader1(); + header1.setStr("s1"); + header1.setNum(100); + RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header1); + cmd.setRemark("remark"); + cmd.setOpaque(1001); + cmd.setVersion(99); + cmd.setLanguage(LanguageCode.JAVA); + cmd.setFlag(3); + cmd.makeCustomHeaderToNet(); + RocketMQSerializable.rocketMQProtocolEncode(cmd, buf); + RemotingCommand cmd2 = RocketMQSerializable.rocketMQProtocolDecode(buf, buf.readableBytes()); + assertThat(cmd2.getRemark()).isEqualTo("remark"); + assertThat(cmd2.getCode()).isEqualTo(code); + assertThat(cmd2.getOpaque()).isEqualTo(1001); + assertThat(cmd2.getVersion()).isEqualTo(99); + assertThat(cmd2.getLanguage()).isEqualTo(LanguageCode.JAVA); + assertThat(cmd2.getFlag()).isEqualTo(3); + + MyHeader1 h2 = (MyHeader1) cmd2.decodeCommandCustomHeader(MyHeader1.class); + assertThat(h2.getStr()).isEqualTo("s1"); + assertThat(h2.getNum()).isEqualTo(100); + } } From dd62ed0f3b16919adec5d5eece21a1050dc9c5a0 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Tue, 29 Oct 2024 20:07:49 +0800 Subject: [PATCH 206/265] [ISSUE #8892] Add test cases to config manager v2 (#8873) * fix: add unit test for TopicConfigManagerV2 Signed-off-by: Li Zhanhui * fix: add unit test cases for config manager v2 Signed-off-by: Li Zhanhui * chore: add copyright header Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui --- .../broker/config/v2/ConfigStorage.java | 14 +- .../config/v2/ConsumerOffsetManagerV2.java | 2 + .../config/v2/SubscriptionGroupManagerV2.java | 2 + .../v2/ConsumerOffsetManagerV2Test.java | 193 ++++++++++++++++++ .../v2/SubscriptionGroupManagerV2Test.java | 141 +++++++++++++ .../config/v2/TopicConfigManagerV2Test.java | 123 +++++++++++ 6 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java index af259aaa374..a31b573daa7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java @@ -27,11 +27,11 @@ import org.apache.rocketmq.common.config.ConfigHelper; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyOptions; -import org.rocksdb.DirectSlice; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; +import org.rocksdb.Slice; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; @@ -112,10 +112,16 @@ public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) { readOptions.setTotalOrderSeek(true); readOptions.setTailing(false); readOptions.setAutoPrefixMode(true); - readOptions.setIterateLowerBound(new DirectSlice(beginKey)); - readOptions.setIterateUpperBound(new DirectSlice(endKey)); + // Use DirectSlice till the follow issue is fixed: + // https://github.com/facebook/rocksdb/issues/13098 + // + // readOptions.setIterateUpperBound(new DirectSlice(endKey)); + byte[] buf = new byte[endKey.remaining()]; + endKey.slice().get(buf); + readOptions.setIterateUpperBound(new Slice(buf)); + RocksIterator iterator = db.newIterator(defaultCFHandle, readOptions); - iterator.seekToFirst(); + iterator.seek(beginKey.slice()); return iterator; } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java index 5b0885c491a..2c5d3677d88 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java @@ -390,10 +390,12 @@ public void commitPullOffset(String clientHost, String group, String topic, int ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId); ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(8); + valueBuf.writeLong(offset); try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); } catch (RocksDBException e) { LOG.error("Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}", group, topic, queueId, offset); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java index 8da6f9d2bc5..f535fa195a9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java @@ -74,6 +74,7 @@ private boolean loadSubscriptions() { if (null != subscriptionGroupConfig) { super.updateSubscriptionGroupConfigWithoutPersist(subscriptionGroupConfig); } + iterator.next(); } } finally { beginKey.release(); @@ -163,6 +164,7 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName writeBatch.delete(ConfigHelper.readBytes(keyBuf)); long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion(); ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + configStorage.write(writeBatch); } catch (RocksDBException e) { log.error("Failed to remove subscription group config by group-name={}", groupName, e); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java new file mode 100644 index 00000000000..d7f46855e1a --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.config.v2; + +import java.io.File; +import java.io.IOException; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ConsumerOffsetManagerV2Test { + + private ConfigStorage configStorage; + + private ConsumerOffsetManagerV2 consumerOffsetManagerV2; + + @Mock + private BrokerController controller; + + @Rule + public TemporaryFolder tf = new TemporaryFolder(); + + @After + public void cleanUp() { + if (null != configStorage) { + configStorage.shutdown(); + } + } + + @Before + public void setUp() throws IOException { + BrokerConfig brokerConfig = new BrokerConfig(); + Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig(); + + File configStoreDir = tf.newFolder(); + configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); + } + + /** + * Verify consumer offset can survive restarts + */ + @Test + public void testCommitOffset_Standard() { + Assert.assertTrue(consumerOffsetManagerV2.load()); + + String clientHost = "localhost"; + String topic = "T1"; + String group = "G0"; + int queueId = 1; + long queueOffset = 100; + consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + + configStorage.shutdown(); + consumerOffsetManagerV2.getOffsetTable().clear(); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + + configStorage.start(); + consumerOffsetManagerV2.load(); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + } + + /** + * Verify commit offset can survive config store restart + */ + @Test + public void testCommitOffset_LMQ() { + Assert.assertTrue(consumerOffsetManagerV2.load()); + + String clientHost = "localhost"; + String topic = MixAll.LMQ_PREFIX + "T1"; + String group = "G0"; + int queueId = 1; + long queueOffset = 100; + consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + + configStorage.shutdown(); + + configStorage.start(); + consumerOffsetManagerV2.load(); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + } + + + /** + * Verify commit offset can survive config store restart + */ + @Test + public void testCommitPullOffset_LMQ() { + Assert.assertTrue(consumerOffsetManagerV2.load()); + + String clientHost = "localhost"; + String topic = MixAll.LMQ_PREFIX + "T1"; + String group = "G0"; + int queueId = 1; + long queueOffset = 100; + consumerOffsetManagerV2.commitPullOffset(clientHost, group, topic, queueId, queueOffset); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId)); + + configStorage.shutdown(); + + configStorage.start(); + consumerOffsetManagerV2.load(); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId)); + } + + /** + * Verify commit offset can survive config store restart + */ + @Test + public void testRemoveByTopicAtGroup() { + Assert.assertTrue(consumerOffsetManagerV2.load()); + + String clientHost = "localhost"; + String topic = MixAll.LMQ_PREFIX + "T1"; + String topic2 = MixAll.LMQ_PREFIX + "T2"; + String group = "G0"; + int queueId = 1; + long queueOffset = 100; + consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset); + consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); + + consumerOffsetManagerV2.removeConsumerOffset(topic + ConsumerOffsetManager.TOPIC_GROUP_SEPARATOR + group); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); + + configStorage.shutdown(); + configStorage.start(); + consumerOffsetManagerV2.load(); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); + } + + /** + * Verify commit offset can survive config store restart + */ + @Test + public void testRemoveByGroup() { + Assert.assertTrue(consumerOffsetManagerV2.load()); + + String clientHost = "localhost"; + String topic = MixAll.LMQ_PREFIX + "T1"; + String topic2 = MixAll.LMQ_PREFIX + "T2"; + String group = "G0"; + int queueId = 1; + long queueOffset = 100; + consumerOffsetManagerV2.commitOffset(clientHost, group, topic, queueId, queueOffset); + consumerOffsetManagerV2.commitOffset(clientHost, group, topic2, queueId, queueOffset); + Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + consumerOffsetManagerV2.removeOffset(group); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); + + configStorage.shutdown(); + configStorage.start(); + consumerOffsetManagerV2.load(); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java new file mode 100644 index 00000000000..6d436a7c4db --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.config.v2; + +import java.io.File; +import java.io.IOException; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicy; +import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.MessageStore; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class SubscriptionGroupManagerV2Test { + private ConfigStorage configStorage; + + private SubscriptionGroupManagerV2 subscriptionGroupManagerV2; + + @Mock + private BrokerController controller; + + @Mock + private MessageStore messageStore; + + @Rule + public TemporaryFolder tf = new TemporaryFolder(); + + @After + public void cleanUp() { + if (null != configStorage) { + configStorage.shutdown(); + } + } + + @Before + public void setUp() throws IOException { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setAutoCreateSubscriptionGroup(false); + Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig(); + + Mockito.doReturn(messageStore).when(controller).getMessageStore(); + Mockito.doReturn(1L).when(messageStore).getStateMachineVersion(); + + File configStoreDir = tf.newFolder(); + configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + configStorage.start(); + subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage); + } + + + @Test + public void testUpdateSubscriptionGroupConfig() { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName("G1"); + subscriptionGroupConfig.setConsumeEnable(true); + subscriptionGroupConfig.setRetryMaxTimes(16); + subscriptionGroupConfig.setGroupSysFlag(1); + GroupRetryPolicy retryPolicy = new GroupRetryPolicy(); + retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL); + subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy); + subscriptionGroupConfig.setBrokerId(1); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + subscriptionGroupConfig.setConsumeMessageOrderly(true); + subscriptionGroupConfig.setConsumeTimeoutMinute(30); + subscriptionGroupConfig.setConsumeFromMinEnable(true); + subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1); + subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true); + subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig); + + SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + Assert.assertEquals(subscriptionGroupConfig, found); + + subscriptionGroupManagerV2.getSubscriptionGroupTable().clear(); + configStorage.shutdown(); + configStorage.start(); + subscriptionGroupManagerV2.load(); + found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + Assert.assertEquals(subscriptionGroupConfig, found); + } + + + @Test + public void testDeleteSubscriptionGroupConfig() { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName("G1"); + subscriptionGroupConfig.setConsumeEnable(true); + subscriptionGroupConfig.setRetryMaxTimes(16); + subscriptionGroupConfig.setGroupSysFlag(1); + GroupRetryPolicy retryPolicy = new GroupRetryPolicy(); + retryPolicy.setType(GroupRetryPolicyType.EXPONENTIAL); + subscriptionGroupConfig.setGroupRetryPolicy(retryPolicy); + subscriptionGroupConfig.setBrokerId(1); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + subscriptionGroupConfig.setConsumeMessageOrderly(true); + subscriptionGroupConfig.setConsumeTimeoutMinute(30); + subscriptionGroupConfig.setConsumeFromMinEnable(true); + subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(1); + subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(true); + subscriptionGroupManagerV2.updateSubscriptionGroupConfig(subscriptionGroupConfig); + + SubscriptionGroupConfig found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + Assert.assertEquals(subscriptionGroupConfig, found); + subscriptionGroupManagerV2.removeSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + + found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + Assert.assertNull(found); + + configStorage.shutdown(); + configStorage.start(); + subscriptionGroupManagerV2.load(); + found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); + Assert.assertNull(found); + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java new file mode 100644 index 00000000000..92c936b110a --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.config.v2; + +import java.io.File; +import java.io.IOException; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + + +@RunWith(value = MockitoJUnitRunner.class) +public class TopicConfigManagerV2Test { + + private ConfigStorage configStorage; + + private TopicConfigManagerV2 topicConfigManagerV2; + + @Mock + private BrokerController controller; + + @Rule + public TemporaryFolder tf = new TemporaryFolder(); + + @After + public void cleanUp() { + if (null != configStorage) { + configStorage.shutdown(); + } + } + + @Before + public void setUp() throws IOException { + BrokerConfig brokerConfig = new BrokerConfig(); + Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + Mockito.doReturn(messageStoreConfig).when(controller).getMessageStoreConfig(); + + File configStoreDir = tf.newFolder(); + configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + configStorage.start(); + topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); + } + + @Test + public void testUpdateTopicConfig() { + TopicConfig topicConfig = new TopicConfig(); + String topicName = "T1"; + topicConfig.setTopicName(topicName); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setWriteQueueNums(4); + topicConfig.setOrder(true); + topicConfig.setTopicSysFlag(4); + topicConfigManagerV2.updateTopicConfig(topicConfig); + + Assert.assertTrue(configStorage.shutdown()); + + topicConfigManagerV2.getTopicConfigTable().clear(); + + Assert.assertTrue(configStorage.start()); + Assert.assertTrue(topicConfigManagerV2.load()); + + TopicConfig loaded = topicConfigManagerV2.selectTopicConfig(topicName); + Assert.assertNotNull(loaded); + Assert.assertEquals(topicName, loaded.getTopicName()); + Assert.assertEquals(6, loaded.getPerm()); + Assert.assertEquals(8, loaded.getReadQueueNums()); + Assert.assertEquals(4, loaded.getWriteQueueNums()); + Assert.assertTrue(loaded.isOrder()); + Assert.assertEquals(4, loaded.getTopicSysFlag()); + + Assert.assertTrue(topicConfigManagerV2.containsTopic(topicName)); + } + + @Test + public void testRemoveTopicConfig() { + TopicConfig topicConfig = new TopicConfig(); + String topicName = "T1"; + topicConfig.setTopicName(topicName); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setWriteQueueNums(4); + topicConfig.setOrder(true); + topicConfig.setTopicSysFlag(4); + topicConfigManagerV2.updateTopicConfig(topicConfig); + topicConfigManagerV2.removeTopicConfig(topicName); + Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName)); + Assert.assertTrue(configStorage.shutdown()); + + Assert.assertTrue(configStorage.start()); + Assert.assertTrue(topicConfigManagerV2.load()); + Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName)); + } +} From b974362b37c581b6bd77bca3dcac35bad71798d5 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 30 Oct 2024 14:20:00 +0800 Subject: [PATCH 207/265] Revert "fix RequestCode overflowed issues:8868 (#8871)" (#8874) This reverts commit 327abe5d6eb0b74f48867821447e2f0718509a42. --- .../remoting/protocol/RequestCode.java | 14 +++---- .../protocol/RocketMQSerializableTest.java | 37 ------------------- 2 files changed, 7 insertions(+), 44 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index 1e10c96f428..cfc5cc22785 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -88,13 +88,13 @@ public class RequestCode { public static final int GET_TIMER_METRICS = 61; - public static final short POP_MESSAGE = 3442; - public static final short ACK_MESSAGE = 3443; - public static final short BATCH_ACK_MESSAGE = 3543; - public static final short PEEK_MESSAGE = 3444; - public static final short CHANGE_MESSAGE_INVISIBLETIME = 3445; - public static final short NOTIFICATION = 3446; - public static final short POLLING_INFO = 3447; + public static final int POP_MESSAGE = 200050; + public static final int ACK_MESSAGE = 200051; + public static final int BATCH_ACK_MESSAGE = 200151; + public static final int PEEK_MESSAGE = 200052; + public static final int CHANGE_MESSAGE_INVISIBLETIME = 200053; + public static final int NOTIFICATION = 200054; + public static final int POLLING_INFO = 200055; public static final int PUT_KV_CONFIG = 100; diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java index c7533bd0b76..7cf32d70c34 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RocketMQSerializableTest.java @@ -214,41 +214,4 @@ public void testFastEncode() throws Exception { assertThat(h2.getStr()).isEqualTo("s1"); assertThat(h2.getNum()).isEqualTo(100); } - - @Test - public void testRequestCodeEncode() throws Exception { - testRequestCodeEncode(RequestCode.POP_MESSAGE); - testRequestCodeEncode(RequestCode.ACK_MESSAGE); - testRequestCodeEncode(RequestCode.BATCH_ACK_MESSAGE); - testRequestCodeEncode(RequestCode.PEEK_MESSAGE); - testRequestCodeEncode(RequestCode.CHANGE_MESSAGE_INVISIBLETIME); - testRequestCodeEncode(RequestCode.NOTIFICATION); - testRequestCodeEncode(RequestCode.POLLING_INFO); - } - - public void testRequestCodeEncode(int code) throws Exception { - ByteBuf buf = ByteBufAllocator.DEFAULT.buffer(16); - MyHeader1 header1 = new MyHeader1(); - header1.setStr("s1"); - header1.setNum(100); - RemotingCommand cmd = RemotingCommand.createRequestCommand(code, header1); - cmd.setRemark("remark"); - cmd.setOpaque(1001); - cmd.setVersion(99); - cmd.setLanguage(LanguageCode.JAVA); - cmd.setFlag(3); - cmd.makeCustomHeaderToNet(); - RocketMQSerializable.rocketMQProtocolEncode(cmd, buf); - RemotingCommand cmd2 = RocketMQSerializable.rocketMQProtocolDecode(buf, buf.readableBytes()); - assertThat(cmd2.getRemark()).isEqualTo("remark"); - assertThat(cmd2.getCode()).isEqualTo(code); - assertThat(cmd2.getOpaque()).isEqualTo(1001); - assertThat(cmd2.getVersion()).isEqualTo(99); - assertThat(cmd2.getLanguage()).isEqualTo(LanguageCode.JAVA); - assertThat(cmd2.getFlag()).isEqualTo(3); - - MyHeader1 h2 = (MyHeader1) cmd2.decodeCommandCustomHeader(MyHeader1.class); - assertThat(h2.getStr()).isEqualTo("s1"); - assertThat(h2.getNum()).isEqualTo(100); - } } From fe8077250b790ea7ff9184ce1752513f9bd3d6de Mon Sep 17 00:00:00 2001 From: Crazywen Date: Wed, 30 Oct 2024 16:42:30 +0800 Subject: [PATCH 208/265] [ISSUE #8875] Fix HAConnection leak --- .../java/org/apache/rocketmq/store/ha/DefaultHAService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java index 75e0afa4e89..c0e203862ca 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/DefaultHAService.java @@ -350,8 +350,8 @@ public void run() { + sc.socket().getRemoteSocketAddress()); try { HAConnection conn = createConnection(sc); - conn.start(); DefaultHAService.this.addConnection(conn); + conn.start(); } catch (Exception e) { log.error("new HAConnection exception", e); sc.close(); From 5600684eb437dd5a4aeb9c658e24200bcb74909b Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Wed, 30 Oct 2024 20:08:28 +0800 Subject: [PATCH 209/265] [ISSUE #8725] clean DefaultMQPushConsumer after start fail (#8726) * [ISSUE #8725]clean DefaultMQPushConsumer after start fail * clean DefaultLitePullConsumerImpl after start fail --- .../impl/consumer/DefaultLitePullConsumerImpl.java | 7 ++++++- .../impl/consumer/DefaultMQPushConsumerImpl.java | 14 ++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 3f90b67ec99..f5ff3179bf0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -307,7 +307,12 @@ public synchronized void start() throws MQClientException { log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup()); - operateAfterRunning(); + try { + operateAfterRunning(); + } catch (Exception e) { + shutdown(); + throw e; + } break; case RUNNING: diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index c92cadf5057..4eccba8e8d4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -1006,10 +1006,16 @@ public synchronized void start() throws MQClientException { break; } - this.updateTopicSubscribeInfoWhenSubscriptionChanged(); - this.mQClientFactory.checkClientInBroker(); - if (this.mQClientFactory.sendHeartbeatToAllBrokerWithLock()) { - this.mQClientFactory.rebalanceImmediately(); + try { + this.updateTopicSubscribeInfoWhenSubscriptionChanged(); + this.mQClientFactory.checkClientInBroker(); + if (this.mQClientFactory.sendHeartbeatToAllBrokerWithLock()) { + this.mQClientFactory.rebalanceImmediately(); + } + } catch (Exception e) { + log.warn("Start the consumer {} fail.", this.defaultMQPushConsumer.getConsumerGroup(), e); + shutdown(); + throw e; } } From 311b831c0223c1c9a875a844e869e19b58dae297 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 6 Nov 2024 10:12:24 +0800 Subject: [PATCH 210/265] [ISSUE #8829] Keep data version while reload and XXXConfigManagerV2 turns off sync Signed-off-by: Li Zhanhui --- .../org/apache/rocketmq/broker/config/v2/ConfigStorage.java | 4 ++-- .../rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java | 2 +- .../rocketmq/broker/config/v2/TopicConfigManagerV2.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java index a31b573daa7..6bc62957a86 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java @@ -87,8 +87,8 @@ protected void initOptions() { protected void initAbleWalWriteOptions() { this.ableWalWriteOptions = new WriteOptions(); - // For metadata, prioritize data integrity - this.ableWalWriteOptions.setSync(true); + // Given that fdatasync is kind of expensive, sync-WAL for every write cannot be afforded. + this.ableWalWriteOptions.setSync(false); // We need WAL for config changes this.ableWalWriteOptions.setDisableWAL(false); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java index f535fa195a9..dea8a2d2c17 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java @@ -72,7 +72,7 @@ private boolean loadSubscriptions() { while (iterator.isValid()) { SubscriptionGroupConfig subscriptionGroupConfig = parseSubscription(iterator.key(), iterator.value()); if (null != subscriptionGroupConfig) { - super.updateSubscriptionGroupConfigWithoutPersist(subscriptionGroupConfig); + super.putSubscriptionGroupConfig(subscriptionGroupConfig); } iterator.next(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java index b1a3d2d85ce..4e36b087275 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java @@ -76,7 +76,7 @@ private boolean loadTopicConfig() { byte[] value = iterator.value(); TopicConfig topicConfig = parseTopicConfig(key, value); if (null != topicConfig) { - super.updateSingleTopicConfigWithoutPersist(topicConfig); + super.putTopicConfig(topicConfig); } iterator.next(); } From 43f3c2b61bf5dbdb174c4664121677d66542277c Mon Sep 17 00:00:00 2001 From: rongtong Date: Wed, 6 Nov 2024 10:15:42 +0800 Subject: [PATCH 211/265] [ISSUE #8885] Resolve the issue of inaccurate CK number statistics (#8886) --- .../apache/rocketmq/broker/processor/PopMessageProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 07bc0ac07b2..fe8ccb03dc0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -815,6 +815,9 @@ private boolean appendCheckPoint(final PopMessageRequestHeader requestHeader, ck.addDiff((int) (msgQueueOffset - offset)); } + this.brokerController.getBrokerStatsManager().incBrokerCkNums(1); + this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1); + final boolean addBufferSuc = this.popBufferMergeService.addCk( ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset() ); From e281697b8e07cfe77eb26ffa46ac76ba3eca75de Mon Sep 17 00:00:00 2001 From: Thomas Lee Date: Wed, 6 Nov 2024 19:30:18 +0800 Subject: [PATCH 212/265] [ISSUE #8808] Resolve unsupported 'UseBiasedLocking' VM Option for JDK21 (#8809) --- distribution/bin/runbroker.cmd | 4 ++-- distribution/bin/runbroker.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd index fefaab3013f..f9a710985a0 100644 --- a/distribution/bin/runbroker.cmd +++ b/distribution/bin/runbroker.cmd @@ -53,8 +53,8 @@ if %JAVA_MAJOR_VERSION% lss 17 ( set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch" set "JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g" - set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking" + set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions" set "JAVA_OPT=%JAVA_OPT% %JAVA_OPT_EXT% -cp "%CLASSPATH%"" ) -"%JAVA%" %JAVA_OPT% %* \ No newline at end of file +"%JAVA%" %JAVA_OPT% %* diff --git a/distribution/bin/runbroker.sh b/distribution/bin/runbroker.sh index e6e2132aba3..e701e6c3200 100644 --- a/distribution/bin/runbroker.sh +++ b/distribution/bin/runbroker.sh @@ -105,7 +105,7 @@ choose_gc_options JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=15g" -JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" +JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking -XX:+IgnoreUnrecognizedVMOptions" #JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" From 851473443e88343c651ac203877330c6cbee3f42 Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Thu, 7 Nov 2024 19:04:54 +0800 Subject: [PATCH 213/265] [ISSUE #8882] Change the compare method for acl signature to improve the security. (#8883) * Change the compare method for acl signature to improve the security. * Change the compare method for acl signature to improve the security. --- .../main/java/org/apache/rocketmq/acl/common/AclUtils.java | 3 +-- .../apache/rocketmq/acl/plain/PlainPermissionManager.java | 5 ++++- .../authentication/chain/DefaultAuthenticationHandler.java | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java index 937619beee4..f32acaf2f74 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java @@ -63,8 +63,7 @@ public static byte[] combineBytes(byte[] b1, byte[] b2) { } public static String calSignature(byte[] data, String secretKey) { - String signature = AclSigner.calSignature(data, secretKey); - return signature; + return AclSigner.calSignature(data, secretKey); } public static void IPv6AddressCheck(String netAddress) { diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index b075e5364ee..daedc38f2e7 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -22,6 +22,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.MessageDigest; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +36,7 @@ import org.apache.rocketmq.acl.PermissionChecker; import org.apache.rocketmq.acl.common.AclConstants; import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclSigner; import org.apache.rocketmq.acl.common.AclUtils; import org.apache.rocketmq.acl.common.Permission; import org.apache.rocketmq.common.AclConfig; @@ -618,7 +620,8 @@ public void validate(PlainAccessResource plainAccessResource) { // Check the signature String signature = AclUtils.calSignature(plainAccessResource.getContent(), ownedAccess.getSecretKey()); - if (!signature.equals(plainAccessResource.getSignature())) { + if (plainAccessResource.getSignature() == null + || !MessageDigest.isEqual(signature.getBytes(AclSigner.DEFAULT_CHARSET), plainAccessResource.getSignature().getBytes(AclSigner.DEFAULT_CHARSET))) { throw new AclException(String.format("Check signature failed for accessKey=%s", plainAccessResource.getAccessKey())); } diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java b/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java index 04f13164507..4b50de756ab 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authentication/chain/DefaultAuthenticationHandler.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.auth.authentication.chain; +import java.security.MessageDigest; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import org.apache.commons.lang3.StringUtils; @@ -62,7 +63,8 @@ protected void doAuthenticate(DefaultAuthenticationContext context, User user) { throw new AuthenticationException("User:{} is disabled.", context.getUsername()); } String signature = AclSigner.calSignature(context.getContent(), user.getPassword()); - if (!StringUtils.equals(signature, context.getSignature())) { + if (context.getSignature() == null + || !MessageDigest.isEqual(signature.getBytes(AclSigner.DEFAULT_CHARSET), context.getSignature().getBytes(AclSigner.DEFAULT_CHARSET))) { throw new AuthenticationException("check signature failed."); } } From dbd2791eca741de29bc8962c66a7744f0c3311a5 Mon Sep 17 00:00:00 2001 From: mawen12 <1181963012mw@gmail.com> Date: Fri, 8 Nov 2024 09:03:10 +0800 Subject: [PATCH 214/265] [ISSUE #8889] handle namespace outside the loop (#8890) * handle namespace outside the loop * fix checkstyle check failed --- .../client/impl/consumer/DefaultLitePullConsumerImpl.java | 8 ++++---- .../client/impl/consumer/DefaultMQPullConsumerImpl.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index f5ff3179bf0..f85dcc7b459 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -1084,12 +1084,12 @@ private void resetTopic(List msgList) { } //If namespace not null , reset Topic without namespace. - for (MessageExt messageExt : msgList) { - if (null != this.defaultLitePullConsumer.getNamespace()) { - messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultLitePullConsumer.getNamespace())); + String namespace = this.defaultLitePullConsumer.getNamespace(); + if (namespace != null) { + for (MessageExt messageExt : msgList) { + messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace)); } } - } public void updateConsumeOffset(MessageQueue mq, long offset) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index c877ccc0702..9a8ea8fb4fe 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -290,12 +290,12 @@ public void resetTopic(List msgList) { } //If namespace not null , reset Topic without namespace. - for (MessageExt messageExt : msgList) { - if (null != this.getDefaultMQPullConsumer().getNamespace()) { - messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultMQPullConsumer.getNamespace())); + String namespace = this.getDefaultMQPullConsumer().getNamespace(); + if (namespace != null) { + for (MessageExt messageExt : msgList) { + messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), namespace)); } } - } public void subscriptionAutomatically(final String topic) { From 7eb16fffec6bf7f118e3373df48e76c7647eb348 Mon Sep 17 00:00:00 2001 From: TianMing2018 <2452146988@qq.com> Date: Mon, 11 Nov 2024 10:00:26 +0800 Subject: [PATCH 215/265] fixed typo of RocketMQ_Example.md (#8905) fixed typo worlds 'finalString' --- docs/cn/RocketMQ_Example.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index 4f08e88154f..77c1bd7b270 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -645,7 +645,7 @@ RocketMQ只定义了一些基本语法来支持这个特性。你也可以很容 只有使用push模式的消费者才能用使用SQL92标准的sql语句，接口如下： ``` -public void subscribe(finalString topic, final MessageSelector messageSelector) +public void subscribe(final String topic, final MessageSelector messageSelector) ``` ### 5.2 使用样例 From 2bbc852361132db8697a5788197aa31f5e89d4a1 Mon Sep 17 00:00:00 2001 From: mawen12 <1181963012mw@gmail.com> Date: Tue, 12 Nov 2024 19:29:21 +0800 Subject: [PATCH 216/265] [ISSUE #8906] Handle string toUpperCase outside the loop --- .../apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java | 3 ++- .../java/org/apache/rocketmq/remoting/netty/TlsHelper.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java index 59342ca3cd2..5a21ec68e5d 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/MultiProtocolTlsHelper.java @@ -102,8 +102,9 @@ private static ClientAuth parseClientAuthMode(String authMode) { return ClientAuth.NONE; } + String authModeUpper = authMode.toUpperCase(); for (ClientAuth clientAuth : ClientAuth.values()) { - if (clientAuth.name().equals(authMode.toUpperCase())) { + if (clientAuth.name().equals(authModeUpper)) { return clientAuth; } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java index 9e73ad7ae0a..2a67512b14e 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/TlsHelper.java @@ -216,8 +216,9 @@ private static ClientAuth parseClientAuthMode(String authMode) { return ClientAuth.NONE; } + String authModeUpper = authMode.toUpperCase(); for (ClientAuth clientAuth : ClientAuth.values()) { - if (clientAuth.name().equals(authMode.toUpperCase())) { + if (clientAuth.name().equals(authModeUpper)) { return clientAuth; } } From 66ba4566f5ebaeac47c73eaaf4a86567e3760063 Mon Sep 17 00:00:00 2001 From: qianye Date: Thu, 14 Nov 2024 14:07:20 +0800 Subject: [PATCH 217/265] close channel when receive go away twice (#8862) close channel when receive go away twice (#8862) --- .../remoting/common/RemotingHelper.java | 30 +++++++---- .../remoting/netty/NettyClientConfig.java | 10 ---- .../remoting/netty/NettyRemotingAbstract.java | 2 +- .../remoting/netty/NettyRemotingClient.java | 53 +++++++------------ 4 files changed, 41 insertions(+), 54 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java index 552fd2b15ff..d94efe71e49 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingHelper.java @@ -21,6 +21,15 @@ import io.netty.channel.ChannelFutureListener; import io.netty.util.Attribute; import io.netty.util.AttributeKey; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.NetworkUtil; @@ -36,15 +45,6 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; -import java.io.IOException; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; -import java.util.HashMap; -import java.util.Map; - public class RemotingHelper { public static final String DEFAULT_CHARSET = "UTF-8"; public static final String DEFAULT_CIDR_ALL = "0.0.0.0/0"; @@ -355,6 +355,18 @@ public void operationComplete(ChannelFuture future) throws Exception { } } + public static CompletableFuture convertChannelFutureToCompletableFuture(ChannelFuture channelFuture) { + CompletableFuture completableFuture = new CompletableFuture<>(); + channelFuture.addListener((ChannelFutureListener) future -> { + if (future.isSuccess()) { + completableFuture.complete(null); + } else { + completableFuture.completeExceptionally(new RemotingConnectException(channelFuture.channel().remoteAddress().toString(), future.cause())); + } + }); + return completableFuture; + } + public static String getRequestCodeDesc(int code) { return REQUEST_CODE_MAP.getOrDefault(code, String.valueOf(code)); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java index 7b7263e27a3..82601636403 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java @@ -59,8 +59,6 @@ public class NettyClientConfig { private boolean enableReconnectForGoAway = true; - private boolean enableTransparentRetry = true; - public boolean isClientCloseSocketIfTimeout() { return clientCloseSocketIfTimeout; } @@ -205,14 +203,6 @@ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { this.enableReconnectForGoAway = enableReconnectForGoAway; } - public boolean isEnableTransparentRetry() { - return enableTransparentRetry; - } - - public void setEnableTransparentRetry(boolean enableTransparentRetry) { - this.enableTransparentRetry = enableTransparentRetry; - } - public String getSocksProxyConfig() { return socksProxyConfig; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index ffa37260594..b0c7099b9dc 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -273,7 +273,7 @@ public void processRequestCommand(final ChannelHandlerContext ctx, final Remotin Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); if (isShuttingDown.get()) { - if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { + if (cmd.getVersion() > MQVersion.Version.V5_3_1.ordinal()) { final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, "please go away"); response.setOpaque(opaque); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index ae82b09edaf..b3042c9f8d3 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -73,6 +73,7 @@ import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.FutureUtils; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -88,6 +89,8 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; +import static org.apache.rocketmq.remoting.common.RemotingHelper.convertChannelFutureToCompletableFuture; + public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); @@ -554,7 +557,7 @@ public RemotingCommand invokeSync(String addr, final RemotingCommand request, lo updateChannelLastResponseTime(addr); return response; } catch (RemotingSendRequestException e) { - LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); + LOGGER.warn("invokeSync: send request exception, so close the channel[addr={}, id={}]", channelRemoteAddr, channel.id()); this.closeChannel(addr, channel); throw e; } catch (RemotingTimeoutException e) { @@ -832,45 +835,27 @@ public CompletableFuture invokeImpl(final Channel channel, final return channelWrapper0; }); if (channelWrapper != null && !channelWrapper.isWrapperOf(channel)) { - if (nettyClientConfig.isEnableTransparentRetry()) { - RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); - retryRequest.setBody(request.getBody()); - retryRequest.setExtFields(request.getExtFields()); - if (channelWrapper.isOK()) { - long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); - stopwatch.stop(); - Channel retryChannel = channelWrapper.getChannel(); - if (retryChannel != null && channel != retryChannel) { - return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); - } - } else { - CompletableFuture future = new CompletableFuture<>(); - ChannelFuture channelFuture = channelWrapper.getChannelFuture(); - channelFuture.addListener(f -> { - long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); - stopwatch.stop(); - if (f.isSuccess()) { - Channel retryChannel0 = channelFuture.channel(); - if (retryChannel0 != null && channel != retryChannel0) { - super.invokeImpl(retryChannel0, retryRequest, timeoutMillis - duration).whenComplete((v, t) -> { - if (t != null) { - future.completeExceptionally(t); - } else { - future.complete(v); - } - }); - } - } else { - future.completeExceptionally(new RemotingConnectException(channelWrapper.channelAddress)); + RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); + retryRequest.setBody(request.getBody()); + retryRequest.setExtFields(request.getExtFields()); + CompletableFuture future = convertChannelFutureToCompletableFuture(channelWrapper.getChannelFuture()); + return future.thenCompose(v -> { + long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); + stopwatch.stop(); + return super.invokeImpl(channelWrapper.getChannel(), retryRequest, timeoutMillis - duration) + .thenCompose(r -> { + if (r.getResponseCommand().getCode() == ResponseCode.GO_AWAY) { + return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr, + new Throwable("Receive GO_AWAY twice in request from channelId=" + channel.id()))); } + return CompletableFuture.completedFuture(r); }); - return future; - } - } + }); } else { LOGGER.warn("invokeImpl receive GO_AWAY, channelWrapper is null or channel is the same in wrapper, channelId={}", channel.id()); } } + return FutureUtils.completeExceptionally(new RemotingSendRequestException(channelRemoteAddr, new Throwable("Receive GO_AWAY from channelId=" + channel.id()))); } return CompletableFuture.completedFuture(responseFuture); }).whenComplete((v, t) -> { From 87b97f271c96bdbb320b1e127cbeaaa4e83c4c2a Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Thu, 14 Nov 2024 19:24:19 +0800 Subject: [PATCH 218/265] [ISSUE #8917]Topic route return none permission message queues for gRPC client (#8919) * When the queue lacks permission, return one queue to allow the client to upload a heartbeat for gRPC Topic route interface. --- .../rocketmq/common/constant/PermName.java | 4 ++++ .../proxy/grpc/v2/route/RouteActivity.java | 12 +++++++++++ .../grpc/v2/route/RouteActivityTest.java | 20 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java index d87461d7f5a..d9a26a2be17 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java @@ -68,4 +68,8 @@ public static boolean isValid(final int perm) { public static boolean isPriority(final int perm) { return (perm & PERM_PRIORITY) == PERM_PRIORITY; } + + public static boolean isAccessible(final int perm) { + return isReadable(perm) || isWriteable(perm); + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java index fe14fe01c64..20ae3aa6c82 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java @@ -244,6 +244,7 @@ protected List genMessageQueueFromQueueData(QueueData queueData, R int r = 0; int w = 0; int rw = 0; + int n = 0; if (PermName.isWriteable(queueData.getPerm()) && PermName.isReadable(queueData.getPerm())) { rw = Math.min(queueData.getWriteQueueNums(), queueData.getReadQueueNums()); r = queueData.getReadQueueNums() - rw; @@ -252,6 +253,8 @@ protected List genMessageQueueFromQueueData(QueueData queueData, R w = queueData.getWriteQueueNums(); } else if (PermName.isReadable(queueData.getPerm())) { r = queueData.getReadQueueNums(); + } else if (!PermName.isAccessible(queueData.getPerm())) { + n = Math.max(1, Math.max(queueData.getWriteQueueNums(), queueData.getReadQueueNums())); } // r here means readOnly queue nums, w means writeOnly queue nums, while rw means both readable and writable queue nums. @@ -283,6 +286,15 @@ protected List genMessageQueueFromQueueData(QueueData queueData, R messageQueueList.add(messageQueue); } + for (int i = 0; i < n; i++) { + MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic) + .setId(queueIdIndex++) + .setPermission(Permission.NONE) + .addAllAcceptMessageTypes(parseTopicMessageType(topicMessageType)) + .build(); + messageQueueList.add(messageQueue); + } + return messageQueueList; } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java index a7ba69098bc..abbf82452ef 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java @@ -272,6 +272,26 @@ public void testGenPartitionFromQueueData() throws Exception { assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); assertEquals(0, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count()); + + // test queueData with 2 read queues, 2 write queues, and no permission, expect 2 no permission queues. + QueueData queueDataWith2R2WNoPerm = createQueueData(2, 2, 0); + List partitionWith2R2WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith2R2WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER); + assertEquals(2, partitionWith2R2WNoPerm.size()); + assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count()); + assertEquals(2, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count()); + assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith2R2WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count()); + + // test queueData with 0 read queues, 0 write queues, and no permission, expect 1 no permission queue. + QueueData queueDataWith0R0WNoPerm = createQueueData(0, 0, 0); + List partitionWith0R0WNoPerm = this.routeActivity.genMessageQueueFromQueueData(queueDataWith0R0WNoPerm, GRPC_TOPIC, TopicMessageType.UNSPECIFIED, GRPC_BROKER); + assertEquals(1, partitionWith0R0WNoPerm.size()); + assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getAcceptMessageTypesValue(0) == MessageType.MESSAGE_TYPE_UNSPECIFIED.getNumber()).count()); + assertEquals(1, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.NONE).count()); + assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith0R0WNoPerm.stream().filter(a -> a.getPermission() == Permission.READ).count()); } private static QueueData createQueueData(int r, int w, int perm) { From 60e68dadd9197730642ce55b1237e6f6f0401aa5 Mon Sep 17 00:00:00 2001 From: Drizzle <464473306@qq.com> Date: Thu, 14 Nov 2024 19:42:25 +0800 Subject: [PATCH 219/265] [ISSUE #8921] Add isWakeCommitWhenPutMessage for AIO Co-authored-by: drizzle.zk --- store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 63022520e2a..378518d249d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -2181,7 +2181,9 @@ public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMess if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) { flushCommitLogService.wakeup(); } else { - commitRealTimeService.wakeup(); + if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) { + commitRealTimeService.wakeup(); + } } } } From 797024f3f8cb51fbe6352832afd38c44f451cf38 Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Fri, 15 Nov 2024 10:35:37 +0800 Subject: [PATCH 220/265] fix update user not valid bug (#8926) --- .../apache/rocketmq/broker/processor/AdminBrokerProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 381889c6247..ac882e94ab0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -3113,7 +3113,7 @@ private RemotingCommand updateUser(ChannelHandlerContext ctx, if (old.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) { throw new AuthenticationException("The super user can only be update by super user"); } - return this.brokerController.getAuthenticationMetadataManager().updateUser(old); + return this.brokerController.getAuthenticationMetadataManager().updateUser(user); }).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS)) .exceptionally(ex -> { LOGGER.error("update user {} error", requestHeader.getUsername(), ex); From 4574647a3a20a17562a0d3fc34b1c8394cc53b8e Mon Sep 17 00:00:00 2001 From: jiao jianan <81030751+jjastan@users.noreply.github.com> Date: Sat, 16 Nov 2024 09:07:55 +0800 Subject: [PATCH 221/265] [ISSUE #8909] Move nullcheck ahead (#8910) Co-authored-by: jiaoja --- .../rocketmq/common/config/AbstractRocksDBStorage.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index d434cce7451..28ed4e924c5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -346,12 +346,13 @@ protected void open(final List cfDescriptors) throws Roc this.db = RocksDB.open(this.options, this.dbPath, cfDescriptors, cfHandles); } assert cfDescriptors.size() == cfHandles.size(); - try (Env env = this.db.getEnv()) { - env.setBackgroundThreads(8, Priority.LOW); - } + if (this.db == null) { throw new RocksDBException("open rocksdb null"); } + try (Env env = this.db.getEnv()) { + env.setBackgroundThreads(8, Priority.LOW); + } } protected abstract boolean postLoad(); From 2e1ca053d2719f2b6ba233496b1b80a48047d403 Mon Sep 17 00:00:00 2001 From: yx9o Date: Mon, 18 Nov 2024 10:19:48 +0800 Subject: [PATCH 222/265] [ISSUE #8901] Add more test coverage for RpcClientImpl (#8902) --- .../remoting/rpc/RpcClientImplTest.java | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java new file mode 100644 index 00000000000..c33511a9764 --- /dev/null +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/rpc/RpcClientImplTest.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.rpc; + +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.header.GetEarliestMsgStoretimeResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.GetMinOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.concurrent.Future; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RpcClientImplTest { + + @Mock + private RemotingClient remotingClient; + + @Mock + private ClientMetadata clientMetadata; + + private RpcClientImpl rpcClient; + + private MessageQueue mq; + + @Mock + private RpcRequest request; + + private final long defaultTimeout = 3000L; + + @Before + public void init() throws IllegalAccessException { + rpcClient = new RpcClientImpl(clientMetadata, remotingClient); + + String defaultBroker = "brokerName"; + mq = new MessageQueue("defaultTopic", defaultBroker, 0); + RpcRequestHeader header = mock(RpcRequestHeader.class); + when(request.getHeader()).thenReturn(header); + when(clientMetadata.getBrokerNameFromMessageQueue(mq)).thenReturn(defaultBroker); + when(clientMetadata.findMasterBrokerAddr(any())).thenReturn("127.0.0.1:10911"); + } + + @Test + public void testInvoke_PULL_MESSAGE() throws Exception { + when(request.getCode()).thenReturn(RequestCode.PULL_MESSAGE); + + doAnswer(invocation -> { + InvokeCallback callback = invocation.getArgument(3); + RemotingCommand response = mock(RemotingCommand.class); + when(response.getBody()).thenReturn("success".getBytes()); + PullMessageResponseHeader responseHeader = mock(PullMessageResponseHeader.class); + when(response.decodeCommandCustomHeader(PullMessageResponseHeader.class)).thenReturn(responseHeader); + callback.operationSucceed(response); + return null; + }).when(remotingClient).invokeAsync( + any(), + any(RemotingCommand.class), + anyLong(), + any(InvokeCallback.class)); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("success", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_GET_MIN_OFFSET() throws Exception { + when(request.getCode()).thenReturn(RequestCode.GET_MIN_OFFSET); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("1".getBytes()); + GetMinOffsetResponseHeader responseHeader = mock(GetMinOffsetResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("1", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_GET_MAX_OFFSET() throws Exception { + when(request.getCode()).thenReturn(RequestCode.GET_MAX_OFFSET); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("1000".getBytes()); + GetMaxOffsetResponseHeader responseHeader = mock(GetMaxOffsetResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("1000", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_SEARCH_OFFSET_BY_TIMESTAMP() throws Exception { + when(request.getCode()).thenReturn(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("1000".getBytes()); + SearchOffsetResponseHeader responseHeader = mock(SearchOffsetResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("1000", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_GET_EARLIEST_MSG_STORETIME() throws Exception { + when(request.getCode()).thenReturn(RequestCode.GET_EARLIEST_MSG_STORETIME); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("10000".getBytes()); + GetEarliestMsgStoretimeResponseHeader responseHeader = mock(GetEarliestMsgStoretimeResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("10000", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_QUERY_CONSUMER_OFFSET() throws Exception { + when(request.getCode()).thenReturn(RequestCode.QUERY_CONSUMER_OFFSET); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("1000".getBytes()); + QueryConsumerOffsetResponseHeader responseHeader = mock(QueryConsumerOffsetResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("1000", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_UPDATE_CONSUMER_OFFSET() throws Exception { + when(request.getCode()).thenReturn(RequestCode.UPDATE_CONSUMER_OFFSET); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + when(responseCommand.getBody()).thenReturn("success".getBytes()); + UpdateConsumerOffsetResponseHeader responseHeader = mock(UpdateConsumerOffsetResponseHeader.class); + when(responseCommand.decodeCommandCustomHeader(UpdateConsumerOffsetResponseHeader.class)).thenReturn(responseHeader); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertEquals("success", new String((byte[]) actual.getBody())); + } + + @Test + public void testInvoke_GET_TOPIC_STATS_INFO() throws Exception { + when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_STATS_INFO); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + when(responseCommand.getBody()).thenReturn(topicStatsTable.encode()); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertTrue(actual.getBody() instanceof TopicStatsTable); + } + + @Test + public void testInvoke_GET_TOPIC_CONFIG() throws Exception { + when(request.getCode()).thenReturn(RequestCode.GET_TOPIC_CONFIG); + + RemotingCommand responseCommand = mock(RemotingCommand.class); + TopicConfigAndQueueMapping topicConfigAndQueueMapping = new TopicConfigAndQueueMapping(); + when(responseCommand.getBody()).thenReturn(RemotingSerializable.encode(topicConfigAndQueueMapping)); + when(remotingClient.invokeSync(any(), any(RemotingCommand.class), anyLong())).thenReturn(responseCommand); + + Future future = rpcClient.invoke(mq, request, defaultTimeout); + RpcResponse actual = future.get(); + + assertEquals(ResponseCode.SUCCESS, actual.getCode()); + assertTrue(actual.getBody() instanceof TopicConfigAndQueueMapping); + } +} From 4e8a5ca48f5d37d8063beb0b79608fd43a942132 Mon Sep 17 00:00:00 2001 From: rongtong Date: Mon, 18 Nov 2024 14:23:00 +0800 Subject: [PATCH 223/265] Add incGroupAckNums and incGroupCkNums to LmqBrokerStatsManager (#8943) --- .../rocketmq/broker/BrokerController.java | 6 ++++- .../store/stats/LmqBrokerStatsManager.java | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index ee211e1b80a..b6c903929d7 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -237,7 +237,7 @@ public class BrokerController { protected final BlockingQueue endTransactionThreadPoolQueue; protected final BlockingQueue adminBrokerThreadPoolQueue; protected final BlockingQueue loadBalanceThreadPoolQueue; - protected final BrokerStatsManager brokerStatsManager; + protected BrokerStatsManager brokerStatsManager; protected final List sendMessageHookList = new ArrayList<>(); protected final List consumeMessageHookList = new ArrayList<>(); protected MessageStore messageStore; @@ -2305,6 +2305,10 @@ public BrokerStatsManager getBrokerStatsManager() { return brokerStatsManager; } + public void setBrokerStatsManager(BrokerStatsManager brokerStatsManager) { + this.brokerStatsManager = brokerStatsManager; + } + public List getSendMessageHookList() { return sendMessageHookList; } diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java index f0e23fe6388..b17fcbc9ca1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java @@ -50,6 +50,33 @@ public void incGroupGetSize(final String group, final String topic, final int in super.incGroupGetSize(lmqGroup, lmqTopic, incValue); } + + @Override + public void incGroupAckNums(final String group, final String topic, final int incValue) { + String lmqGroup = group; + String lmqTopic = topic; + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } + super.incGroupAckNums(lmqGroup, lmqTopic, incValue); + } + + @Override + public void incGroupCkNums(final String group, final String topic, final int incValue) { + String lmqGroup = group; + String lmqTopic = topic; + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } + super.incGroupCkNums(lmqGroup, lmqTopic, incValue); + } + @Override public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) { String lmqGroup = group; From 163451ed30c3f3cd3029d7b6c8b77d65d163dcee Mon Sep 17 00:00:00 2001 From: rongtong Date: Mon, 18 Nov 2024 14:30:01 +0800 Subject: [PATCH 224/265] Support for Persisting LMQ Consumer Offsets in Config V1 Using RocksDB (#8939) --- .../rocketmq/broker/BrokerController.java | 3 +- .../v1/RocksDBLmqConsumerOffsetManager.java | 103 ------------------ .../RocksDBLmqConsumerOffsetManagerTest.java | 55 +++------- 3 files changed, 19 insertions(+), 142 deletions(-) delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index b6c903929d7..143922e456f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -77,7 +77,6 @@ import org.apache.rocketmq.broker.offset.ConsumerOrderInfoManager; import org.apache.rocketmq.broker.offset.LmqConsumerOffsetManager; import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; -import org.apache.rocketmq.broker.config.v1.RocksDBLmqConsumerOffsetManager; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin; import org.apache.rocketmq.broker.processor.AckMessageProcessor; @@ -352,7 +351,7 @@ public BrokerController( } else if (this.messageStoreConfig.isEnableRocksDBStore()) { this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); - this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); + this.consumerOffsetManager = new RocksDBConsumerOffsetManager(this); } else { this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new LmqTopicConfigManager(this) : new TopicConfigManager(this); this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new LmqSubscriptionGroupManager(this) : new SubscriptionGroupManager(this); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java deleted file mode 100644 index e961c6c635a..00000000000 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v1/RocksDBLmqConsumerOffsetManager.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.broker.config.v1; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; - -public class RocksDBLmqConsumerOffsetManager extends RocksDBConsumerOffsetManager { - private ConcurrentHashMap lmqOffsetTable = new ConcurrentHashMap<>(512); - - public RocksDBLmqConsumerOffsetManager(BrokerController brokerController) { - super(brokerController); - } - - @Override - public long queryOffset(final String group, final String topic, final int queueId) { - if (!MixAll.isLmq(group)) { - return super.queryOffset(group, topic, queueId); - } - // topic@group - String key = topic + TOPIC_GROUP_SEPARATOR + group; - Long offset = lmqOffsetTable.get(key); - if (offset != null) { - return offset; - } - return -1; - } - - @Override - public Map queryOffset(final String group, final String topic) { - if (!MixAll.isLmq(group)) { - return super.queryOffset(group, topic); - } - Map map = new HashMap<>(); - // topic@group - String key = topic + TOPIC_GROUP_SEPARATOR + group; - Long offset = lmqOffsetTable.get(key); - if (offset != null) { - map.put(0, offset); - } - return map; - } - - @Override - public void commitOffset(final String clientHost, final String group, final String topic, final int queueId, - final long offset) { - if (!MixAll.isLmq(group)) { - super.commitOffset(clientHost, group, topic, queueId, offset); - return; - } - // topic@group - String key = topic + TOPIC_GROUP_SEPARATOR + group; - lmqOffsetTable.put(key, offset); - } - - @Override - public String encode() { - return this.encode(false); - } - - @Override - public void decode(String jsonString) { - if (jsonString != null) { - RocksDBLmqConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, RocksDBLmqConsumerOffsetManager.class); - if (obj != null) { - super.setOffsetTable(obj.getOffsetTable()); - this.lmqOffsetTable = obj.lmqOffsetTable; - } - } - } - - @Override - public String encode(final boolean prettyFormat) { - return RemotingSerializable.toJson(this, prettyFormat); - } - - public ConcurrentHashMap getLmqOffsetTable() { - return lmqOffsetTable; - } - - public void setLmqOffsetTable(ConcurrentHashMap lmqOffsetTable) { - this.lmqOffsetTable = lmqOffsetTable; - } -} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java index 1b9916d6ac1..aa5003fc103 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManagerTest.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.broker.offset; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.config.v1.RocksDBLmqConsumerOffsetManager; +import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.store.config.MessageStoreConfig; @@ -28,45 +28,37 @@ import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; import static org.mockito.Mockito.when; public class RocksDBLmqConsumerOffsetManagerTest { private static final String LMQ_GROUP = MixAll.LMQ_PREFIX + "FooBarGroup"; private static final String NON_LMQ_GROUP = "nonLmqGroup"; - private static final String TOPIC = "FooBarTopic"; + + private static final String LMQ_TOPIC = MixAll.LMQ_PREFIX + "FooBarTopic"; + private static final String NON_LMQ_TOPIC = "FooBarTopic"; private static final int QUEUE_ID = 0; private static final long OFFSET = 12345; private BrokerController brokerController; - private RocksDBLmqConsumerOffsetManager offsetManager; + private RocksDBConsumerOffsetManager offsetManager; @Before public void setUp() { brokerController = Mockito.mock(BrokerController.class); when(brokerController.getMessageStoreConfig()).thenReturn(Mockito.mock(MessageStoreConfig.class)); - when(brokerController.getBrokerConfig()).thenReturn(Mockito.mock(BrokerConfig.class)); - offsetManager = new RocksDBLmqConsumerOffsetManager(brokerController); + when(brokerController.getBrokerConfig()).thenReturn(new BrokerConfig()); + offsetManager = new RocksDBConsumerOffsetManager(brokerController); } - @Test - public void testQueryOffsetForLmq() { - // Setup - offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); - // Execute - long actualOffset = offsetManager.queryOffset(LMQ_GROUP, TOPIC, QUEUE_ID); - // Verify - assertEquals("Offset should match the expected value.", OFFSET, actualOffset); - } @Test public void testQueryOffsetForNonLmq() { - long actualOffset = offsetManager.queryOffset(NON_LMQ_GROUP, TOPIC, QUEUE_ID); + long actualOffset = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID); // Verify assertEquals("Offset should not be null.", -1, actualOffset); } @@ -74,10 +66,10 @@ public void testQueryOffsetForNonLmq() { @Test public void testQueryOffsetForLmqGroupWithExistingOffset() { - offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); + offsetManager.commitOffset("127.0.0.1",LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET); // Act - Map actualOffsets = offsetManager.queryOffset(LMQ_GROUP, TOPIC); + Map actualOffsets = offsetManager.queryOffset(LMQ_GROUP, LMQ_TOPIC); // Assert assertNotNull(actualOffsets); @@ -89,23 +81,20 @@ public void testQueryOffsetForLmqGroupWithExistingOffset() { public void testQueryOffsetForLmqGroupWithoutExistingOffset() { // Act Map actualOffsets = offsetManager.queryOffset(LMQ_GROUP, "nonExistingTopic"); - // Assert - assertNotNull(actualOffsets); - assertTrue("The map should be empty for non-existing offsets", actualOffsets.isEmpty()); + assertNull(actualOffsets); } @Test public void testQueryOffsetForNonLmqGroup() { - when(brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep()).thenReturn(1L); // Arrange Map mockOffsets = new HashMap<>(); mockOffsets.put(QUEUE_ID, OFFSET); - offsetManager.commitOffset("clientHost", NON_LMQ_GROUP, TOPIC, QUEUE_ID, OFFSET); + offsetManager.commitOffset("clientHost", NON_LMQ_GROUP, NON_LMQ_TOPIC, QUEUE_ID, OFFSET); // Act - Map actualOffsets = offsetManager.queryOffset(NON_LMQ_GROUP, TOPIC); + Map actualOffsets = offsetManager.queryOffset(NON_LMQ_GROUP, NON_LMQ_TOPIC); // Assert assertNotNull(actualOffsets); @@ -115,21 +104,13 @@ public void testQueryOffsetForNonLmqGroup() { @Test public void testCommitOffsetForLmq() { // Execute - offsetManager.commitOffset("clientHost", LMQ_GROUP, TOPIC, QUEUE_ID, OFFSET); + offsetManager.commitOffset("clientHost", LMQ_GROUP, LMQ_TOPIC, QUEUE_ID, OFFSET); // Verify - Long expectedOffset = offsetManager.getLmqOffsetTable().get(getKey()); + Long expectedOffset = offsetManager.getOffsetTable().get(getLMQKey()).get(QUEUE_ID); assertEquals("Offset should be updated correctly.", OFFSET, expectedOffset.longValue()); } - @Test - public void testEncode() { - offsetManager.setLmqOffsetTable(new ConcurrentHashMap<>(512)); - offsetManager.getLmqOffsetTable().put(getKey(), OFFSET); - String encodedData = offsetManager.encode(); - assertTrue(encodedData.contains(String.valueOf(OFFSET))); - } - - private String getKey() { - return TOPIC + "@" + LMQ_GROUP; + private String getLMQKey() { + return LMQ_TOPIC + "@" + LMQ_GROUP; } } From c961edd2fd3f5c2c4907c1ed6be953be2d8a2b4f Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:46:34 +0800 Subject: [PATCH 225/265] [ISSUE #8945] Remove unnecessary operations from the critical section Co-authored-by: wanghuaiyuan --- .../apache/rocketmq/store/dledger/DLedgerCommitLog.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index e617343f9ad..5f4ef08374c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -568,15 +568,16 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner AppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(msg); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status))); + } + String topicQueueKey = msg.getTopic() + "-" + msg.getQueueId(); topicQueueLock.lock(topicQueueKey); try { defaultMessageStore.assignOffset(msg); - encodeResult = this.messageSerializer.serialize(msg); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status))); - } putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long elapsedTimeInLock; long queueOffset; From bf2f2a77748961ed25e72b69732abe55ab2fd64a Mon Sep 17 00:00:00 2001 From: rongtong Date: Mon, 18 Nov 2024 17:25:20 +0800 Subject: [PATCH 226/265] Fix incorrect path for exportMetadataInRocksDBCommand (#8941) --- .../command/export/ExportMetadataInRocksDBCommand.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java index c466490b8a8..d5726985e3c 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java @@ -77,8 +77,11 @@ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) t return; } - String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); - path += "/" + configType; + String configType = commandLine.getOptionValue("configType").trim(); + if (!path.endsWith("/")) { + path += "/"; + } + path += configType; boolean jsonEnable = false; if (commandLine.hasOption("jsonEnable")) { From 78575af0b8ddd409ea71facceb87c144951269ae Mon Sep 17 00:00:00 2001 From: Ji Juntao Date: Tue, 19 Nov 2024 17:35:10 +0800 Subject: [PATCH 227/265] [ISSUE #8935] Fix behind metrics unit error in timer message store (#8936) --- .../java/org/apache/rocketmq/store/timer/TimerMessageStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 32075474b99..071b1c02192 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1724,7 +1724,7 @@ public long getEnqueueBehindMessages() { public long getEnqueueBehindMillis() { if (System.currentTimeMillis() - lastEnqueueButExpiredTime < 2000) { - return (System.currentTimeMillis() - lastEnqueueButExpiredStoreTime) / 1000; + return System.currentTimeMillis() - lastEnqueueButExpiredStoreTime; } return 0; } From ae7179d75e11f469d68be05fbf556fde42c8a795 Mon Sep 17 00:00:00 2001 From: yuz10 <845238369@qq.com> Date: Wed, 20 Nov 2024 11:10:01 +0800 Subject: [PATCH 228/265] [ISSUE #8765] fix low performance of delay message when enable rocksdb consume queue (#8766) * #7538 fix wrong cachedMsgSize if msg body is changed in consumer callback * [ISSUE #8765] fix low performance of delay message when enable rocksdb consume queue * remove prefetch --- .../store/queue/RocksDBConsumeQueue.java | 74 ++++++++++++++++--- .../store/queue/RocksDBConsumeQueueTest.java | 73 ++++++++++++++++++ 2 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java index 83ba7bebad0..7bd3c2e3057 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java @@ -271,22 +271,17 @@ public long getMinOffsetInQueue() { private int pullNum(long cqOffset, long maxCqOffset) { long diffLong = maxCqOffset - cqOffset; if (diffLong < Integer.MAX_VALUE) { - int diffInt = (int) diffLong; - return diffInt > 16 ? 16 : diffInt; + return (int) diffLong; } - return 16; + return Integer.MAX_VALUE; } @Override public ReferredIterator iterateFrom(final long startIndex) { - try { - long maxCqOffset = getMaxOffsetInQueue(); - if (startIndex < maxCqOffset) { - int num = pullNum(startIndex, maxCqOffset); - return iterateFrom0(startIndex, num); - } - } catch (RocksDBException e) { - log.error("[RocksDBConsumeQueue] iterateFrom error!", e); + long maxCqOffset = getMaxOffsetInQueue(); + if (startIndex < maxCqOffset) { + int num = pullNum(startIndex, maxCqOffset); + return new LargeRocksDBConsumeQueueIterator(startIndex, num); } return null; } @@ -428,4 +423,61 @@ public CqUnit nextAndRelease() { } } } + + private class LargeRocksDBConsumeQueueIterator implements ReferredIterator { + private final long startIndex; + private final int totalCount; + private int currentIndex; + + public LargeRocksDBConsumeQueueIterator(final long startIndex, final int num) { + this.startIndex = startIndex; + this.totalCount = num; + this.currentIndex = 0; + } + + @Override + public boolean hasNext() { + return this.currentIndex < this.totalCount; + } + + + @Override + public CqUnit next() { + if (!hasNext()) { + return null; + } + + final ByteBuffer byteBuffer; + try { + byteBuffer = messageStore.getQueueStore().get(topic, queueId, startIndex + currentIndex); + } catch (RocksDBException e) { + ERROR_LOG.error("get cq from rocksdb failed. topic: {}, queueId: {}", topic, queueId, e); + return null; + } + if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) { + return null; + } + CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong()); + this.currentIndex++; + return cqUnit; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + @Override + public void release() { + } + + @Override + public CqUnit nextAndRelease() { + try { + return next(); + } finally { + release(); + } + } + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java new file mode 100644 index 00000000000..b907ce59519 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.nio.ByteBuffer; + +import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueTable.CQ_UNIT_SIZE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RocksDBConsumeQueueTest extends QueueTestBase { + + @Test + public void testIterator() throws Exception { + if (MixAll.isMac()) { + return; + } + DefaultMessageStore messageStore = mock(DefaultMessageStore.class); + RocksDBConsumeQueueStore rocksDBConsumeQueueStore = mock(RocksDBConsumeQueueStore.class); + when(messageStore.getQueueStore()).thenReturn(rocksDBConsumeQueueStore); + when(rocksDBConsumeQueueStore.getMaxOffsetInQueue(anyString(), anyInt())).thenReturn(10000L); + when(rocksDBConsumeQueueStore.get(anyString(), anyInt(), anyLong())).then(new Answer() { + @Override + public ByteBuffer answer(InvocationOnMock mock) throws Throwable { + long startIndex = mock.getArgument(2); + final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_UNIT_SIZE); + long phyOffset = startIndex * 10; + byteBuffer.putLong(phyOffset); + byteBuffer.putInt(1); + byteBuffer.putLong(0); + byteBuffer.putLong(0); + byteBuffer.flip(); + return byteBuffer; + } + }); + + RocksDBConsumeQueue consumeQueue = new RocksDBConsumeQueue(messageStore, "topic", 0); + ReferredIterator it = consumeQueue.iterateFrom(9000); + for (int i = 0; i < 1000; i++) { + assertTrue(it.hasNext()); + CqUnit next = it.next(); + assertEquals(9000 + i, next.getQueueOffset()); + assertEquals(10 * (9000 + i), next.getPos()); + } + assertFalse(it.hasNext()); + } +} \ No newline at end of file From c13f051eb8deaecc00859028f2e9f398db32b454 Mon Sep 17 00:00:00 2001 From: hqbfz <125714719+3424672656@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:13:12 +0800 Subject: [PATCH 229/265] Improve IO for asynchronous delivery processes (#8954) Co-authored-by: wanghuaiyuan --- .../java/org/apache/rocketmq/store/CommitLog.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 378518d249d..7cf97465512 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -2179,7 +2179,9 @@ public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMess // Asynchronous flush else { if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) { - flushCommitLogService.wakeup(); + if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) { + flushCommitLogService.wakeup(); + } } else { if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) { commitRealTimeService.wakeup(); @@ -2206,9 +2208,13 @@ public CompletableFuture handleDiskFlush(AppendMessageResult r // Asynchronous flush else { if (!CommitLog.this.defaultMessageStore.isTransientStorePoolEnable()) { - flushCommitLogService.wakeup(); + if (defaultMessageStore.getMessageStoreConfig().isWakeFlushWhenPutMessage()) { + flushCommitLogService.wakeup(); + } } else { - commitRealTimeService.wakeup(); + if (defaultMessageStore.getMessageStoreConfig().isWakeCommitWhenPutMessage()) { + commitRealTimeService.wakeup(); + } } return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK); } From 8505482c0b05b6dceb2e3a372bd8c9848c26c244 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Wed, 20 Nov 2024 14:55:30 +0800 Subject: [PATCH 230/265] fix: avoid memory overhead when there is large number of LMQ ConsumeQueue (#8956) --- .../rocketmq/store/queue/AbstractConsumeQueueStore.java | 6 +++++- .../rocketmq/store/queue/RocksDBConsumeQueueStore.java | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java index dfce665d8fa..ef693dc1e65 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java @@ -39,7 +39,11 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte public AbstractConsumeQueueStore(DefaultMessageStore messageStore) { this.messageStore = messageStore; this.messageStoreConfig = messageStore.getMessageStoreConfig(); - this.consumeQueueTable = new ConcurrentHashMap<>(32); + if (messageStoreConfig.isEnableLmq()) { + this.consumeQueueTable = new ConcurrentHashMap<>(32_768); + } else { + this.consumeQueueTable = new ConcurrentHashMap<>(32); + } } @Override diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 67a00157431..0242ec23094 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -480,7 +480,13 @@ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException { public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { ConcurrentMap map = this.consumeQueueTable.get(topic); if (null == map) { - ConcurrentMap newMap = new ConcurrentHashMap<>(128); + ConcurrentMap newMap; + if (MixAll.isLmq(topic)) { + // For LMQ, no need to over allocate internal hashtable + newMap = new ConcurrentHashMap<>(1, 1.0F); + } else { + newMap = new ConcurrentHashMap<>(8); + } ConcurrentMap oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap); if (oldMap != null) { map = oldMap; From 23cc24cc6e2fa33b9be2c434c42dee3a54e726a4 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Wed, 20 Nov 2024 14:57:35 +0800 Subject: [PATCH 231/265] [ISSUE #8947] Notify pop request before calculate consumer lag (#8949) --- .../NotifyMessageArrivingListener.java | 4 +- .../longpolling/PopCommandCallback.java | 49 +++++++++++++++ .../longpolling/PopLongPollingService.java | 49 ++++++++++++--- .../broker/metrics/ConsumerLagCalculator.java | 62 ++++++++++++------- .../processor/NotificationProcessor.java | 4 +- .../broker/processor/PopMessageProcessor.java | 18 +++++- .../PopLongPollingServiceTest.java | 11 ++-- .../apache/rocketmq/common/BrokerConfig.java | 19 ++++++ .../rocketmq/remoting/CommandCallback.java | 22 +++++++ .../remoting/protocol/RemotingCommand.java | 11 ++++ 10 files changed, 206 insertions(+), 43 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java index 1ddb9f4f8e6..9c0ee89e4db 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java @@ -40,8 +40,8 @@ public void arriving(String topic, int queueId, long logicOffset, long tagsCode, this.pullRequestHoldService.notifyMessageArriving( topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties); this.popMessageProcessor.notifyMessageArriving( - topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties); this.notificationProcessor.notifyMessageArriving( - topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties); } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java new file mode 100644 index 00000000000..2e190e20f92 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.longpolling; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import org.apache.rocketmq.broker.metrics.ConsumerLagCalculator; +import org.apache.rocketmq.remoting.CommandCallback; + +public class PopCommandCallback implements CommandCallback { + + private final BiConsumer> biConsumer; + + private final ConsumerLagCalculator.ProcessGroupInfo info; + private final Consumer lagRecorder; + + + public PopCommandCallback( + BiConsumer> biConsumer, + ConsumerLagCalculator.ProcessGroupInfo info, + Consumer lagRecorder) { + + this.biConsumer = biConsumer; + this.info = info; + this.lagRecorder = lagRecorder; + } + + @Override + public void accept() { + biConsumer.accept(info, lagRecorder); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java index b5179114f37..91185fbe94c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java @@ -19,6 +19,7 @@ import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import io.netty.channel.ChannelHandlerContext; +import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -31,6 +32,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.CommandCallback; import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.RequestTask; @@ -45,6 +47,7 @@ import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_TIMEOUT; public class PopLongPollingService extends ServiceThread { + private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); private final BrokerController brokerController; @@ -150,10 +153,10 @@ public void run() { } public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId) { - this.notifyMessageArrivingWithRetryTopic(topic, queueId, null, 0L, null, null); + this.notifyMessageArrivingWithRetryTopic(topic, queueId, -1L, null, 0L, null, null); } - public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, + public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, long offset, Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { String notifyTopic; if (KeyBuilder.isPopRetryTopicV2(topic)) { @@ -161,25 +164,37 @@ public void notifyMessageArrivingWithRetryTopic(final String topic, final int qu } else { notifyTopic = topic; } - notifyMessageArriving(notifyTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + notifyMessageArriving(notifyTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); } - public void notifyMessageArriving(final String topic, final int queueId, + public void notifyMessageArriving(final String topic, final int queueId, long offset, Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { ConcurrentHashMap cids = topicCidMap.get(topic); if (cids == null) { return; } + long interval = brokerController.getBrokerConfig().getPopLongPollingForceNotifyInterval(); + boolean force = interval > 0L && offset % interval == 0L; for (Map.Entry cid : cids.entrySet()) { if (queueId >= 0) { - notifyMessageArriving(topic, -1, cid.getKey(), tagsCode, msgStoreTime, filterBitMap, properties); + notifyMessageArriving(topic, -1, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties); } - notifyMessageArriving(topic, queueId, cid.getKey(), tagsCode, msgStoreTime, filterBitMap, properties); + notifyMessageArriving(topic, queueId, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties); } } public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + return notifyMessageArriving(topic, queueId, cid, false, tagsCode, msgStoreTime, filterBitMap, properties, null); + } + + public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + return notifyMessageArriving(topic, queueId, cid, force, tagsCode, msgStoreTime, filterBitMap, properties, null); + } + + public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force, + Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties, CommandCallback callback) { ConcurrentSkipListSet remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, queueId)); if (remotingCommands == null || remotingCommands.isEmpty()) { return false; @@ -190,7 +205,7 @@ public boolean notifyMessageArriving(final String topic, final int queueId, fina return false; } - if (popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) { + if (!force && popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) { boolean match = popRequest.getMessageFilter().isMatchedByConsumeQueue(tagsCode, new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap)); if (match && properties != null) { @@ -206,16 +221,30 @@ public boolean notifyMessageArriving(final String topic, final int queueId, fina if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("lock release, new msg arrive, wakeUp: {}", popRequest); } - return wakeUp(popRequest); + + return wakeUp(popRequest, callback); } public boolean wakeUp(final PopRequest request) { + return wakeUp(request, null); + } + + public boolean wakeUp(final PopRequest request, CommandCallback callback) { if (request == null || !request.complete()) { return false; } + + if (callback != null && request.getRemotingCommand() != null) { + if (request.getRemotingCommand().getCallbackList() == null) { + request.getRemotingCommand().setCallbackList(new ArrayList<>()); + } + request.getRemotingCommand().getCallbackList().add(callback); + } + if (!request.getCtx().channel().isActive()) { return false; } + Runnable run = () -> { try { final RemotingCommand response = processor.processRequest(request.getCtx(), request.getRemotingCommand()); @@ -234,7 +263,9 @@ public boolean wakeUp(final PopRequest request) { POP_LOGGER.error("ExecuteRequestWhenWakeup run", e1); } }; - this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), request.getRemotingCommand())); + + this.brokerController.getPullMessageExecutor().submit( + new RequestTask(run, request.getChannel(), request.getRemotingCommand())); return true; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java index 3ac6528b2a4..1b898f95de3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java @@ -26,6 +26,8 @@ import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; +import org.apache.rocketmq.broker.longpolling.PopCommandCallback; +import org.apache.rocketmq.broker.longpolling.PopLongPollingService; import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.broker.processor.PopBufferMergeService; import org.apache.rocketmq.broker.processor.PopInflightMessageCounter; @@ -51,6 +53,7 @@ import org.apache.rocketmq.store.exception.ConsumeQueueException; public class ConsumerLagCalculator { + private final BrokerConfig brokerConfig; private final TopicConfigManager topicConfigManager; private final ConsumerManager consumerManager; @@ -59,6 +62,7 @@ public class ConsumerLagCalculator { private final SubscriptionGroupManager subscriptionGroupManager; private final MessageStore messageStore; private final PopBufferMergeService popBufferMergeService; + private final PopLongPollingService popLongPollingService; private final PopInflightMessageCounter popInflightMessageCounter; private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -72,10 +76,11 @@ public ConsumerLagCalculator(BrokerController brokerController) { this.subscriptionGroupManager = brokerController.getSubscriptionGroupManager(); this.messageStore = brokerController.getMessageStore(); this.popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService(); + this.popLongPollingService = brokerController.getPopMessageProcessor().getPopLongPollingService(); this.popInflightMessageCounter = brokerController.getPopInflightMessageCounter(); } - private static class ProcessGroupInfo { + public static class ProcessGroupInfo { public String group; public String topic; public boolean isPop; @@ -211,34 +216,44 @@ public void calculateLag(Consumer lagRecorder) { return; } - CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false); + if (info.isPop && brokerConfig.isEnableNotifyBeforePopCalculateLag()) { + if (popLongPollingService.notifyMessageArriving(info.topic, -1, info.group, + true, null, 0, null, null, new PopCommandCallback(this::calculate, info, lagRecorder))) { + return; + } + } + + calculate(info, lagRecorder); + }); + } + public void calculate(ProcessGroupInfo info, Consumer lagRecorder) { + CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false); + try { + Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); + if (lag != null) { + result.lag = lag.getObject1(); + result.earliestUnconsumedTimestamp = lag.getObject2(); + } + lagRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get lag stats", e); + } + + if (info.isPop) { try { - Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); - if (lag != null) { - result.lag = lag.getObject1(); - result.earliestUnconsumedTimestamp = lag.getObject2(); + Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); + + result = new CalculateLagResult(info.group, info.topic, true); + if (retryLag != null) { + result.lag = retryLag.getObject1(); + result.earliestUnconsumedTimestamp = retryLag.getObject2(); } lagRecorder.accept(result); } catch (ConsumeQueueException e) { LOGGER.error("Failed to get lag stats", e); } - - if (info.isPop) { - try { - Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); - - result = new CalculateLagResult(info.group, info.topic, true); - if (retryLag != null) { - result.lag = retryLag.getObject1(); - result.earliestUnconsumedTimestamp = retryLag.getObject2(); - } - lagRecorder.accept(result); - } catch (ConsumeQueueException e) { - LOGGER.error("Failed to get lag stats", e); - } - } - }); + } } public void calculateInflight(Consumer inflightRecorder) { @@ -320,6 +335,9 @@ public Pair getConsumerLagStats(String group, String topic, boolean earliestUnconsumedTimestamp = 0L; } + LOGGER.debug("GetConsumerLagStats, topic={}, group={}, lag={}, latency={}", topic, group, total, + earliestUnconsumedTimestamp > 0 ? System.currentTimeMillis() - earliestUnconsumedTimestamp : 0); + return new Pair<>(total, earliestUnconsumedTimestamp); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java index 75c77b6d79f..b4ebd9c4a99 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java @@ -62,10 +62,10 @@ public boolean rejectRequest() { // When a new message is written to CommitLog, this method would be called. // Suspended long polling will receive notification and be wakeup. - public void notifyMessageArriving(final String topic, final int queueId, + public void notifyMessageArriving(final String topic, final int queueId, long offset, Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { this.popLongPollingService.notifyMessageArrivingWithRetryTopic( - topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); } public void notifyMessageArriving(final String topic, final int queueId) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index fe8ccb03dc0..e0454afa3ca 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -65,6 +65,7 @@ import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.CommandCallback; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; @@ -97,8 +98,10 @@ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT; public class PopMessageProcessor implements NettyRequestProcessor { + private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); + private final BrokerController brokerController; private final Random random = new Random(System.currentTimeMillis()); String reviveTopic; @@ -196,15 +199,15 @@ public void notifyLongPollingRequestIfNeed(String topic, String group, int queue } } - public void notifyMessageArriving(final String topic, final int queueId, + public void notifyMessageArriving(final String topic, final int queueId, long offset, Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { popLongPollingService.notifyMessageArrivingWithRetryTopic( - topic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); } public void notifyMessageArriving(final String topic, final int queueId, final String cid) { popLongPollingService.notifyMessageArriving( - topic, queueId, cid, null, 0L, null, null); + topic, queueId, cid, false, null, 0L, null, null); } @Override @@ -419,6 +422,15 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC final RemotingCommand finalResponse = response; SubscriptionData finalSubscriptionData = subscriptionData; getMessageFuture.thenApply(restNum -> { + try { + if (request.getCallbackList() != null) { + request.getCallbackList().forEach(CommandCallback::accept); + request.getCallbackList().clear(); + } + } catch (Throwable t) { + POP_LOGGER.error("PopProcessor execute callback error", t); + } + if (!getMessageResult.getMessageBufferList().isEmpty()) { finalResponse.setCode(ResponseCode.SUCCESS); getMessageResult.setStatus(GetMessageStatus.FOUND); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java index 6527beeb682..1f064ec05d1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/longpolling/PopLongPollingServiceTest.java @@ -80,21 +80,22 @@ public void init() { @Test public void testNotifyMessageArrivingWithRetryTopic() { int queueId = 0; - doNothing().when(popLongPollingService).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, null, 0L, null, null); + doNothing().when(popLongPollingService).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null); popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId); - verify(popLongPollingService, times(1)).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, null, 0L, null, null); + verify(popLongPollingService, times(1)).notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, -1L, null, 0L, null, null); } @Test public void testNotifyMessageArriving() { int queueId = 0; Long tagsCode = 123L; + long offset = 123L; long msgStoreTime = System.currentTimeMillis(); byte[] filterBitMap = new byte[]{0x01}; Map properties = new ConcurrentHashMap<>(); - doNothing().when(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); - popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); - verify(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, tagsCode, msgStoreTime, filterBitMap, properties); + doNothing().when(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); + popLongPollingService.notifyMessageArrivingWithRetryTopic(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); + verify(popLongPollingService).notifyMessageArriving(defaultTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties); } @Test diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index c6510476617..f459abf0db2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -227,6 +227,9 @@ public class BrokerConfig extends BrokerIdentity { private int popCkMaxBufferSize = 200000; private int popCkOffsetMaxQueueSize = 20000; private boolean enablePopBatchAck = false; + // set the interval to the maxFilterMessageSize in MessageStoreConfig divided by the cq unit size + private long popLongPollingForceNotifyInterval = 800; + private boolean enableNotifyBeforePopCalculateLag = true; private boolean enableNotifyAfterPopOrderLockRelease = true; private boolean initPopOffsetByCheckMsgInMem = true; // read message from pop retry topic v1, for the compatibility, will be removed in the future version @@ -1326,6 +1329,22 @@ public void setEnableNetWorkFlowControl(boolean enableNetWorkFlowControl) { this.enableNetWorkFlowControl = enableNetWorkFlowControl; } + public long getPopLongPollingForceNotifyInterval() { + return popLongPollingForceNotifyInterval; + } + + public void setPopLongPollingForceNotifyInterval(long popLongPollingForceNotifyInterval) { + this.popLongPollingForceNotifyInterval = popLongPollingForceNotifyInterval; + } + + public boolean isEnableNotifyBeforePopCalculateLag() { + return enableNotifyBeforePopCalculateLag; + } + + public void setEnableNotifyBeforePopCalculateLag(boolean enableNotifyBeforePopCalculateLag) { + this.enableNotifyBeforePopCalculateLag = enableNotifyBeforePopCalculateLag; + } + public boolean isEnableNotifyAfterPopOrderLockRelease() { return enableNotifyAfterPopOrderLockRelease; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java new file mode 100644 index 00000000000..884f3d9e5d1 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/CommandCallback.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting; + +public interface CommandCallback { + + void accept(); +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java index 5de48350cf0..9b2b0f07b4f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -38,6 +39,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.CommandCallback; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -96,6 +98,7 @@ public class RemotingCommand { private transient byte[] body; private boolean suspended; private transient Stopwatch processTimer; + private transient List callbackList; protected RemotingCommand() { } @@ -639,4 +642,12 @@ public Stopwatch getProcessTimer() { public void setProcessTimer(Stopwatch processTimer) { this.processTimer = processTimer; } + + public List getCallbackList() { + return callbackList; + } + + public void setCallbackList(List callbackList) { + this.callbackList = callbackList; + } } From 9202de34c30db004db26f4976f165595af1b8bd3 Mon Sep 17 00:00:00 2001 From: Humkum <1109939087@qq.com> Date: Wed, 20 Nov 2024 14:58:30 +0800 Subject: [PATCH 232/265] [ISSUE #8933] feat: DefaultPullConsumer add balance switch (#8934) --- .../client/consumer/DefaultMQPullConsumer.java | 10 ++++++++++ .../impl/consumer/DefaultMQPullConsumerImpl.java | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 7c9a65ecdbf..9e7a86d9b49 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -88,6 +88,8 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume private int maxReconsumeTimes = 16; + private boolean enableRebalance = true; + public DefaultMQPullConsumer() { this(MixAll.DEFAULT_CONSUMER_GROUP, null); } @@ -468,4 +470,12 @@ public void setMaxReconsumeTimes(final int maxReconsumeTimes) { public void persist(MessageQueue mq) { this.getOffsetStore().persist(queueWithNamespace(mq)); } + + public boolean isEnableRebalance() { + return enableRebalance; + } + + public void setEnableRebalance(boolean enableRebalance) { + this.enableRebalance = enableRebalance; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 9a8ea8fb4fe..e05c614c6d2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -381,6 +381,9 @@ public Set subscriptions() { @Override public void doRebalance() { + if (!defaultMQPullConsumer.isEnableRebalance()) { + return; + } if (this.rebalanceImpl != null) { this.rebalanceImpl.doRebalance(false); } @@ -388,6 +391,10 @@ public void doRebalance() { @Override public boolean tryRebalance() { + if (!defaultMQPullConsumer.isEnableRebalance()) { + return true; + } + if (this.rebalanceImpl != null) { return this.rebalanceImpl.doRebalance(false); } From 796c95b9283af301438cdba8582c2ff6f286a679 Mon Sep 17 00:00:00 2001 From: qianye Date: Thu, 21 Nov 2024 16:40:48 +0800 Subject: [PATCH 233/265] [ISSUE #8929] Proxy adds message body empty check when send in grpc protocol (#8930) --- WORKSPACE | 2 +- pom.xml | 20 +++++++++---------- .../rocketmq/proxy/config/ProxyConfig.java | 12 +++++++++++ .../grpc/v2/producer/SendMessageActivity.java | 5 +++++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 9b06bc63413..9125a67f88b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -71,7 +71,7 @@ maven_install( "org.bouncycastle:bcpkix-jdk15on:1.69", "com.google.code.gson:gson:2.8.9", "com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2", - "org.apache.rocketmq:rocketmq-proto:2.0.3", + "org.apache.rocketmq:rocketmq-proto:2.0.4", "com.google.protobuf:protobuf-java:3.20.1", "com.google.protobuf:protobuf-java-util:3.20.1", "com.conversantmedia:disruptor:1.2.10", diff --git a/pom.xml b/pom.xml index 33db3c7f486..ddc8fc81b68 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 6.0.53 1.0-beta-4 1.4.2 - 2.0.3 + 2.0.4 1.53.0 3.20.1 1.2.10 @@ -641,16 +641,8 @@ ${rocketmq-proto.version} - io.grpc - grpc-protobuf - - - io.grpc - grpc-stub - - - io.grpc - grpc-netty-shaded + * + * @@ -1097,6 +1089,12 @@ + + + jakarta.annotation + jakarta.annotation-api + 1.3.5 + diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java index 9901c8ea1fa..3b09b1388fa 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java @@ -103,6 +103,10 @@ public class ProxyConfig implements ConfigFile { * max message body size, 0 or negative number means no limit for proxy */ private int maxMessageSize = 4 * 1024 * 1024; + /** + * if true, proxy will check message body size and reject msg if it's body is empty + */ + private boolean enableMessageBodyEmptyCheck = true; /** * max user property size, 0 or negative number means no limit for proxy */ @@ -1525,4 +1529,12 @@ public boolean isEnableBatchAck() { public void setEnableBatchAck(boolean enableBatchAck) { this.enableBatchAck = enableBatchAck; } + + public boolean isEnableMessageBodyEmptyCheck() { + return enableMessageBodyEmptyCheck; + } + + public void setEnableMessageBodyEmptyCheck(boolean enableMessageBodyEmptyCheck) { + this.enableMessageBodyEmptyCheck = enableMessageBodyEmptyCheck; + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java index 8679bfbe388..8a3d315c68c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java @@ -132,6 +132,11 @@ protected int buildSysFlag(apache.rocketmq.v2.Message protoMessage) { } protected void validateMessageBodySize(ByteString body) { + if (ConfigurationManager.getProxyConfig().isEnableMessageBodyEmptyCheck()) { + if (body.isEmpty()) { + throw new GrpcProxyException(Code.MESSAGE_BODY_EMPTY, "message body cannot be empty"); + } + } int max = ConfigurationManager.getProxyConfig().getMaxMessageSize(); if (max <= 0) { return; From 5f2642391dad0ec4043c6984a9b8b038f10f89b9 Mon Sep 17 00:00:00 2001 From: qianye Date: Thu, 21 Nov 2024 17:53:46 +0800 Subject: [PATCH 234/265] [ISSUE #8877] Refactor lock in ReceiptHandleGroup to make the lock can be properly released when future can not be completed (#8916) --- .../proxy/common/ReceiptHandleGroup.java | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java index 6fee38d117b..15da628dc3c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java @@ -25,14 +25,19 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.proxy.config.ConfigurationManager; public class ReceiptHandleGroup { + protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); // The messages having the same messageId will be deduplicated based on the parameters of broker, queueId, and offset protected final Map> receiptHandleMap = new ConcurrentHashMap<>(); @@ -98,6 +103,7 @@ public long getOffset() { public static class HandleData { private final Semaphore semaphore = new Semaphore(1); + private final AtomicLong lastLockTimeMs = new AtomicLong(-1L); private volatile boolean needRemove = false; private volatile MessageReceiptHandle messageReceiptHandle; @@ -105,15 +111,40 @@ public HandleData(MessageReceiptHandle messageReceiptHandle) { this.messageReceiptHandle = messageReceiptHandle; } - public boolean lock(long timeoutMs) { + public Long lock(long timeoutMs) { try { - return this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS); + boolean result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS); + long currentTimeMs = System.currentTimeMillis(); + if (result) { + this.lastLockTimeMs.set(currentTimeMs); + return currentTimeMs; + } else { + // if the lock is expired, can be acquired again + long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 3; + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + synchronized (this) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + log.warn("HandleData lock expired, acquire lock success and reset lock time. " + + "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); + this.lastLockTimeMs.set(currentTimeMs); + return currentTimeMs; + } + } + } + } + return null; } catch (InterruptedException e) { - return false; + return null; } } - public void unlock() { + public void unlock(long lockTimeMs) { + // if the lock is expired, we don't need to unlock it + if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 2) { + log.warn("HandleData lock expired, unlock fail. MessageReceiptHandle={}, lockTime={}, now={}", + messageReceiptHandle, lockTimeMs, System.currentTimeMillis()); + return; + } this.semaphore.release(); } @@ -149,7 +180,8 @@ public void put(String msgID, MessageReceiptHandle value) { if (handleData == null || handleData.needRemove) { return new HandleData(value); } - if (!handleData.lock(timeout)) { + Long lockTimeMs = handleData.lock(timeout); + if (lockTimeMs == null) { throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to put handle failed"); } try { @@ -158,7 +190,7 @@ public void put(String msgID, MessageReceiptHandle value) { } handleData.messageReceiptHandle = value; } finally { - handleData.unlock(); + handleData.unlock(lockTimeMs); } return handleData; }); @@ -176,7 +208,8 @@ public MessageReceiptHandle get(String msgID, String handle) { long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup(); AtomicReference res = new AtomicReference<>(); handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> { - if (!handleData.lock(timeout)) { + Long lockTimeMs = handleData.lock(timeout); + if (lockTimeMs == null) { throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to get handle failed"); } try { @@ -185,7 +218,7 @@ public MessageReceiptHandle get(String msgID, String handle) { } res.set(handleData.messageReceiptHandle); } finally { - handleData.unlock(); + handleData.unlock(lockTimeMs); } return handleData; }); @@ -200,7 +233,8 @@ public MessageReceiptHandle remove(String msgID, String handle) { long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup(); AtomicReference res = new AtomicReference<>(); handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> { - if (!handleData.lock(timeout)) { + Long lockTimeMs = handleData.lock(timeout); + if (lockTimeMs == null) { throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to remove and get handle failed"); } try { @@ -210,7 +244,7 @@ public MessageReceiptHandle remove(String msgID, String handle) { } return null; } finally { - handleData.unlock(); + handleData.unlock(lockTimeMs); } }); removeHandleMapKeyIfNeed(msgID); @@ -240,7 +274,8 @@ public void computeIfPresent(String msgID, String handle, } long timeout = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup(); handleMap.computeIfPresent(new HandleKey(handle), (handleKey, handleData) -> { - if (!handleData.lock(timeout)) { + Long lockTimeMs = handleData.lock(timeout); + if (lockTimeMs == null) { throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, "try to compute failed"); } CompletableFuture future = function.apply(handleData.messageReceiptHandle); @@ -255,7 +290,7 @@ public void computeIfPresent(String msgID, String handle, handleData.messageReceiptHandle = messageReceiptHandle; } } finally { - handleData.unlock(); + handleData.unlock(lockTimeMs); } if (handleData.needRemove) { handleMap.remove(handleKey, handleData); From e876bed084ca9d642011a9d77b82c7f52b582500 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Fri, 22 Nov 2024 11:02:50 +0800 Subject: [PATCH 235/265] [ISSUE #8955] Fix message buffer not release and dispatch thread exit in tiered storage (#8965) --- .../core/MessageStoreDispatcherImpl.java | 40 ++++++--- .../tieredstore/index/IndexStoreFile.java | 88 +++++++++---------- .../provider/PosixFileSegment.java | 3 +- 3 files changed, 72 insertions(+), 59 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java index 982909c5ee5..9b1e53564d7 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java @@ -92,8 +92,10 @@ public void dispatchWithSemaphore(FlatFileInterface flatFile) { semaphore.acquire(); this.doScheduleDispatch(flatFile, false) .whenComplete((future, throwable) -> semaphore.release()); - } catch (InterruptedException e) { + } catch (Throwable t) { semaphore.release(); + log.error("MessageStore dispatch error, topic={}, queueId={}", + flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t); } } @@ -156,8 +158,7 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, } if (currentOffset < minOffsetInQueue) { - log.warn("MessageDispatcher#dispatch, current offset is too small, " + - "topic={}, queueId={}, offset={}-{}, current={}", + log.warn("MessageDispatcher#dispatch, current offset is too small, topic={}, queueId={}, offset={}-{}, current={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset); flatFileStore.destroyFile(flatFile.getMessageQueue()); flatFileStore.computeIfAbsent(new MessageQueue(topic, brokerName, queueId)); @@ -165,16 +166,14 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, } if (currentOffset > maxOffsetInQueue) { - log.warn("MessageDispatcher#dispatch, current offset is too large, " + - "topic: {}, queueId: {}, offset={}-{}, current={}", + log.warn("MessageDispatcher#dispatch, current offset is too large, topic={}, queueId={}, offset={}-{}, current={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset); return CompletableFuture.completedFuture(false); } long interval = TimeUnit.HOURS.toMillis(storeConfig.getCommitLogRollingInterval()); if (flatFile.rollingFile(interval)) { - log.info("MessageDispatcher#dispatch, rolling file, " + - "topic: {}, queueId: {}, offset={}-{}, current={}", + log.info("MessageDispatcher#dispatch, rolling file, topic={}, queueId={}, offset={}-{}, current={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset); } @@ -189,8 +188,20 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, ConsumeQueueInterface consumeQueue = defaultStore.getConsumeQueue(topic, queueId); CqUnit cqUnit = consumeQueue.get(currentOffset); + if (cqUnit == null) { + log.warn("MessageDispatcher#dispatch cq not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}", + topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset); + return CompletableFuture.completedFuture(false); + } + SelectMappedBufferResult message = defaultStore.selectOneMessageByOffset(cqUnit.getPos(), cqUnit.getSize()); + if (message == null) { + log.warn("MessageDispatcher#dispatch message not found, topic={}, queueId={}, offset={}-{}, current={}, remain={}", + topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset); + return CompletableFuture.completedFuture(false); + } + boolean timeout = MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) + storeConfig.getTieredStoreGroupCommitTimeout() < System.currentTimeMillis(); boolean bufferFull = maxOffsetInQueue - currentOffset > storeConfig.getTieredStoreGroupCommitCount(); @@ -198,6 +209,7 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, if (!timeout && !bufferFull && !force) { log.debug("MessageDispatcher#dispatch hold, topic={}, queueId={}, offset={}-{}, current={}, remain={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset); + message.release(); return CompletableFuture.completedFuture(false); } else { if (MessageFormatUtil.getStoreTimeStamp(message.getByteBuffer()) + @@ -205,11 +217,11 @@ public CompletableFuture doScheduleDispatch(FlatFileInterface flatFile, log.warn("MessageDispatcher#dispatch behind too much, topic={}, queueId={}, offset={}-{}, current={}, remain={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset); } else { - log.info("MessageDispatcher#dispatch, topic={}, queueId={}, offset={}-{}, current={}, remain={}", + log.info("MessageDispatcher#dispatch success, topic={}, queueId={}, offset={}-{}, current={}, remain={}", topic, queueId, minOffsetInQueue, maxOffsetInQueue, currentOffset, maxOffsetInQueue - currentOffset); } + message.release(); } - message.release(); long offset = currentOffset; for (; offset < targetOffset; offset++) { @@ -279,7 +291,7 @@ public CompletableFuture commitAsync(FlatFileInterface flatFile) { } flatFile.release(); } - }, MessageStoreExecutor.getInstance().bufferCommitExecutor); + }, storeExecutor.bufferCommitExecutor); } /** @@ -301,8 +313,12 @@ public void constructIndexFile(long topicId, DispatchRequest request) { public void run() { log.info("{} service started", this.getServiceName()); while (!this.isStopped()) { - flatFileStore.deepCopyFlatFileToList().forEach(this::dispatchWithSemaphore); - this.waitForRunning(Duration.ofSeconds(20).toMillis()); + try { + flatFileStore.deepCopyFlatFileToList().forEach(this::dispatchWithSemaphore); + this.waitForRunning(Duration.ofSeconds(20).toMillis()); + } catch (Throwable t) { + log.error("MessageStore dispatch error", t); + } } log.info("{} service shutdown", this.getServiceName()); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java index f9604b43e6f..25cd634873d 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java @@ -38,7 +38,6 @@ import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.apache.rocketmq.store.logfile.MappedFile; import org.apache.rocketmq.tieredstore.MessageStoreConfig; -import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.AppendResult; import org.apache.rocketmq.tieredstore.provider.FileSegment; import org.apache.rocketmq.tieredstore.provider.PosixFileSegment; @@ -261,56 +260,55 @@ public CompletableFuture> queryAsync( protected CompletableFuture> queryAsyncFromUnsealedFile( String key, int maxCount, long beginTime, long endTime) { - return CompletableFuture.supplyAsync(() -> { - List result = new ArrayList<>(); - try { - fileReadWriteLock.readLock().lock(); - if (!UNSEALED.equals(this.fileStatus.get()) && !SEALED.equals(this.fileStatus.get())) { - return result; - } + List result = new ArrayList<>(); + try { + fileReadWriteLock.readLock().lock(); + if (!UNSEALED.equals(this.fileStatus.get()) && !SEALED.equals(this.fileStatus.get())) { + return CompletableFuture.completedFuture(result); + } - if (mappedFile == null || !mappedFile.hold()) { - return result; - } + if (mappedFile == null || !mappedFile.hold()) { + return CompletableFuture.completedFuture(result); + } - int hashCode = this.hashCode(key); - int slotPosition = this.getSlotPosition(hashCode % this.hashSlotMaxCount); - int slotValue = this.getSlotValue(slotPosition); + int hashCode = this.hashCode(key); + int slotPosition = this.getSlotPosition(hashCode % this.hashSlotMaxCount); + int slotValue = this.getSlotValue(slotPosition); - int left = MAX_QUERY_COUNT; - while (left > 0 && - slotValue > INVALID_INDEX && - slotValue <= this.indexItemCount.get()) { + int left = MAX_QUERY_COUNT; + while (left > 0 && + slotValue > INVALID_INDEX && + slotValue <= this.indexItemCount.get()) { - byte[] bytes = new byte[IndexItem.INDEX_ITEM_SIZE]; - ByteBuffer buffer = this.byteBuffer.duplicate(); - buffer.position(this.getItemPosition(slotValue)); - buffer.get(bytes); - IndexItem indexItem = new IndexItem(bytes); - long storeTimestamp = indexItem.getTimeDiff() + beginTimestamp.get(); - if (hashCode == indexItem.getHashCode() && - beginTime <= storeTimestamp && storeTimestamp <= endTime) { - result.add(indexItem); - if (result.size() > maxCount) { - break; - } + byte[] bytes = new byte[IndexItem.INDEX_ITEM_SIZE]; + ByteBuffer buffer = this.byteBuffer.duplicate(); + buffer.position(this.getItemPosition(slotValue)); + buffer.get(bytes); + IndexItem indexItem = new IndexItem(bytes); + long storeTimestamp = indexItem.getTimeDiff() + beginTimestamp.get(); + if (hashCode == indexItem.getHashCode() && + beginTime <= storeTimestamp && storeTimestamp <= endTime) { + result.add(indexItem); + if (result.size() > maxCount) { + break; } - slotValue = indexItem.getItemIndex(); - left--; } - - log.debug("IndexStoreFile query from unsealed mapped file, timestamp: {}, result size: {}, " + - "key: {}, hashCode: {}, maxCount: {}, timestamp={}-{}", - getTimestamp(), result.size(), key, hashCode, maxCount, beginTime, endTime); - } catch (Exception e) { - log.error("IndexStoreFile query from unsealed mapped file error, timestamp: {}, " + - "key: {}, maxCount: {}, timestamp={}-{}", getTimestamp(), key, maxCount, beginTime, endTime, e); - } finally { - fileReadWriteLock.readLock().unlock(); - mappedFile.release(); + slotValue = indexItem.getItemIndex(); + left--; } - return result; - }, MessageStoreExecutor.getInstance().bufferFetchExecutor); + + log.debug("IndexStoreFile query from unsealed mapped file, timestamp: {}, result size: {}, " + + "key: {}, hashCode: {}, maxCount: {}, timestamp={}-{}", + getTimestamp(), result.size(), key, hashCode, maxCount, beginTime, endTime); + } catch (Exception e) { + log.error("IndexStoreFile query from unsealed mapped file error, timestamp: {}, " + + "key: {}, maxCount: {}, timestamp={}-{}", getTimestamp(), key, maxCount, beginTime, endTime, e); + } finally { + fileReadWriteLock.readLock().unlock(); + mappedFile.release(); + } + + return CompletableFuture.completedFuture(result); } protected CompletableFuture> queryAsyncFromSegmentFile( @@ -465,7 +463,7 @@ public void shutdown() { fileReadWriteLock.writeLock().lock(); this.fileStatus.set(IndexStatusEnum.SHUTDOWN); if (this.fileSegment != null && this.fileSegment instanceof PosixFileSegment) { - ((PosixFileSegment) this.fileSegment).close(); + this.fileSegment.close(); } if (this.mappedFile != null) { this.mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10)); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java index fb150c928cf..656af2ba1c6 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java @@ -30,7 +30,6 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.tieredstore.MessageStoreConfig; -import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream; @@ -230,6 +229,6 @@ public CompletableFuture commit0( return false; } return true; - }, MessageStoreExecutor.getInstance().bufferCommitExecutor); + }); } } From 715dd5a885ae89ebc05aea33971029d7306c80ae Mon Sep 17 00:00:00 2001 From: rongtong Date: Fri, 22 Nov 2024 12:57:38 +0800 Subject: [PATCH 236/265] Adding the EnableLmqStats option allows monitoring of LMQ statistics at runtime (#8973) --- .../rocketmq/broker/BrokerController.java | 2 +- .../apache/rocketmq/common/BrokerConfig.java | 11 ++ .../store/stats/LmqBrokerStatsManager.java | 117 +++++++++++------- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 143922e456f..b907489bbfb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -341,7 +341,7 @@ public BrokerController( this.messageStoreConfig = messageStoreConfig; this.authConfig = authConfig; this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); - this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); + this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); this.broadcastOffsetManager = new BroadcastOffsetManager(this); if (ConfigManagerVersion.V2.getVersion().equals(brokerConfig.getConfigManagerVersion())) { this.configStorage = new ConfigStorage(messageStoreConfig.getStorePathRootDir()); diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index f459abf0db2..9d8d9135217 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -435,6 +435,9 @@ public class BrokerConfig extends BrokerIdentity { private boolean appendCkAsync = false; + + private boolean enableLmqStats = false; + /** * V2 is recommended in cases where LMQ feature is extensively used. */ @@ -1905,6 +1908,14 @@ public void setAppendCkAsync(boolean appendCkAsync) { this.appendCkAsync = appendCkAsync; } + public boolean isEnableLmqStats() { + return enableLmqStats; + } + + public void setEnableLmqStats(boolean enableLmqStats) { + this.enableLmqStats = enableLmqStats; + } + public String getConfigManagerVersion() { return configManagerVersion; } diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java index b17fcbc9ca1..20ed8793318 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java @@ -16,23 +16,29 @@ */ package org.apache.rocketmq.store.stats; +import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; public class LmqBrokerStatsManager extends BrokerStatsManager { - public LmqBrokerStatsManager(String clusterName, boolean enableQueueStat) { - super(clusterName, enableQueueStat); + private final BrokerConfig brokerConfig; + + public LmqBrokerStatsManager(BrokerConfig brokerConfig) { + super(brokerConfig.getBrokerClusterName(), brokerConfig.isEnableDetailStat()); + this.brokerConfig = brokerConfig; } @Override public void incGroupGetNums(final String group, final String topic, final int incValue) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incGroupGetNums(lmqGroup, lmqTopic, incValue); } @@ -41,25 +47,28 @@ public void incGroupGetNums(final String group, final String topic, final int in public void incGroupGetSize(final String group, final String topic, final int incValue) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incGroupGetSize(lmqGroup, lmqTopic, incValue); } - @Override public void incGroupAckNums(final String group, final String topic, final int incValue) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incGroupAckNums(lmqGroup, lmqTopic, incValue); } @@ -68,11 +77,13 @@ public void incGroupAckNums(final String group, final String topic, final int in public void incGroupCkNums(final String group, final String topic, final int incValue) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incGroupCkNums(lmqGroup, lmqTopic, incValue); } @@ -81,11 +92,13 @@ public void incGroupCkNums(final String group, final String topic, final int inc public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incGroupGetLatency(lmqGroup, lmqTopic, queueId, incValue); } @@ -94,11 +107,13 @@ public void incGroupGetLatency(final String group, final String topic, final int public void incSendBackNums(final String group, final String topic) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.incSendBackNums(lmqGroup, lmqTopic); } @@ -107,11 +122,13 @@ public void incSendBackNums(final String group, final String topic) { public double tpsGroupGetNums(final String group, final String topic) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } return super.tpsGroupGetNums(lmqGroup, lmqTopic); } @@ -121,11 +138,13 @@ public void recordDiskFallBehindTime(final String group, final String topic, fin final long fallBehind) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.recordDiskFallBehindTime(lmqGroup, lmqTopic, queueId, fallBehind); } @@ -135,11 +154,13 @@ public void recordDiskFallBehindSize(final String group, final String topic, fin final long fallBehind) { String lmqGroup = group; String lmqTopic = topic; - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; + if (!brokerConfig.isEnableLmqStats()) { + if (MixAll.isLmq(group)) { + lmqGroup = MixAll.LMQ_PREFIX; + } + if (MixAll.isLmq(topic)) { + lmqTopic = MixAll.LMQ_PREFIX; + } } super.recordDiskFallBehindSize(lmqGroup, lmqTopic, queueId, fallBehind); } From a8779c0d4e815835bc17f708a0215cb5877b4004 Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Fri, 22 Nov 2024 15:34:36 +0800 Subject: [PATCH 237/265] [ISSUE #8961] Automatic recognition of address scheme in Topic Route by host (#8962) * automatic recognition of address scheme in topic route by host. --- .../rocketmq/common/utils/IPAddressUtils.java | 8 +++ .../apache/rocketmq/proxy/common/Address.java | 20 +++++++ .../activity/GetTopicRouteActivity.java | 2 +- .../service/route/ProxyTopicRouteData.java | 4 +- .../rocketmq/proxy/common/AddressTest.java | 60 +++++++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java index ca66bc93be2..5133219d9cd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/IPAddressUtils.java @@ -35,6 +35,14 @@ public static boolean isValidIp(String ip) { return VALIDATOR.isValid(ip); } + public static boolean isValidIPv4(String ip) { + return VALIDATOR.isValidInet4Address(ip); + } + + public static boolean isValidIPv6(String ip) { + return VALIDATOR.isValidInet6Address(ip); + } + public static boolean isValidCidr(String cidr) { return isValidIPv4Cidr(cidr) || isValidIPv6Cidr(cidr); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java index 2fc1dab40ed..1f247194e2a 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java @@ -18,6 +18,7 @@ import com.google.common.net.HostAndPort; import java.util.Objects; +import org.apache.rocketmq.common.utils.IPAddressUtils; public class Address { @@ -31,6 +32,11 @@ public enum AddressScheme { private AddressScheme addressScheme; private HostAndPort hostAndPort; + public Address(HostAndPort hostAndPort) { + this.addressScheme = buildScheme(hostAndPort); + this.hostAndPort = hostAndPort; + } + public Address(AddressScheme addressScheme, HostAndPort hostAndPort) { this.addressScheme = addressScheme; this.hostAndPort = hostAndPort; @@ -52,6 +58,20 @@ public void setHostAndPort(HostAndPort hostAndPort) { this.hostAndPort = hostAndPort; } + private AddressScheme buildScheme(HostAndPort hostAndPort) { + if (hostAndPort == null) { + return AddressScheme.UNRECOGNIZED; + } + String address = hostAndPort.getHost(); + if (IPAddressUtils.isValidIPv4(address)) { + return AddressScheme.IPv4; + } + if (IPAddressUtils.isValidIPv6(address)) { + return AddressScheme.IPv6; + } + return AddressScheme.DOMAIN_NAME; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java index 9972c26c991..56ec34fae6a 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/GetTopicRouteActivity.java @@ -50,7 +50,7 @@ protected RemotingCommand processRequest0(ChannelHandlerContext ctx, RemotingCom (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); List
    addressList = new ArrayList<>(); // AddressScheme is just a placeholder and will not affect topic route result in this case. - addressList.add(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort()))); + addressList.add(new Address(HostAndPort.fromParts(proxyConfig.getRemotingAccessAddr(), proxyConfig.getRemotingListenPort()))); ProxyTopicRouteData proxyTopicRouteData = messagingProcessor.getTopicRouteDataForProxy(context, addressList, requestHeader.getTopic()); TopicRouteData topicRouteData = proxyTopicRouteData.buildTopicRouteData(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java index b5e65818ac5..4c33580adaf 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java @@ -43,7 +43,7 @@ public ProxyTopicRouteData(TopicRouteData topicRouteData) { brokerData.getBrokerAddrs().forEach((brokerId, brokerAddr) -> { HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr); - proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, brokerHostAndPort))); + proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(brokerHostAndPort))); }); this.brokerDatas.add(proxyBrokerData); } @@ -61,7 +61,7 @@ public ProxyTopicRouteData(TopicRouteData topicRouteData, int port) { HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr); HostAndPort proxyHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), port); - proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, proxyHostAndPort))); + proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(proxyHostAndPort))); }); this.brokerDatas.add(proxyBrokerData); } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java new file mode 100644 index 00000000000..b0df5bafc14 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/AddressTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +import com.google.common.net.HostAndPort; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +public class AddressTest { + + @Test + public void testConstructorWithIPv4() { + HostAndPort hostAndPort = HostAndPort.fromString("192.168.1.1:8080"); + Address address = new Address(hostAndPort); + + assertEquals(Address.AddressScheme.IPv4, address.getAddressScheme()); + assertEquals(hostAndPort, address.getHostAndPort()); + } + + @Test + public void testConstructorWithIPv6() { + HostAndPort hostAndPort = HostAndPort.fromString("[2001:db8::1]:8080"); + Address address = new Address(hostAndPort); + + assertEquals(Address.AddressScheme.IPv6, address.getAddressScheme()); + assertEquals(hostAndPort, address.getHostAndPort()); + } + + @Test + public void testConstructorWithDomainName() { + HostAndPort hostAndPort = HostAndPort.fromString("example.com:8080"); + Address address = new Address(hostAndPort); + + assertEquals(Address.AddressScheme.DOMAIN_NAME, address.getAddressScheme()); + assertEquals(hostAndPort, address.getHostAndPort()); + } + + @Test + public void testConstructorWithNullHostAndPort() { + Address address = new Address(null); + + assertEquals(Address.AddressScheme.UNRECOGNIZED, address.getAddressScheme()); + assertNull(address.getHostAndPort()); + } +} \ No newline at end of file From b638d4cfbbdfcbd6a8aa2feee580519f68d85730 Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Mon, 25 Nov 2024 19:24:43 +0800 Subject: [PATCH 238/265] [ISSUE #8460] Set default broker name when revive found ack without broker name field (#8981) --- .../rocketmq/broker/processor/PopReviveService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index f27934efdfd..e1ead86169b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -28,6 +28,7 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; @@ -376,7 +377,9 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { } AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class); PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId); - String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + ackMsg.getBrokerName(); + String brokerName = StringUtils.isNotBlank(ackMsg.getBrokerName()) ? + ackMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName(); + String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + brokerName; PopCheckPoint point = map.get(mergeKey); if (point == null) { if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) { @@ -401,7 +404,9 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) { BatchAckMsg bAckMsg = JSON.parseObject(raw, BatchAckMsg.class); PopMetricsManager.incPopReviveAckGetCount(bAckMsg, queueId); - String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + bAckMsg.getBrokerName(); + String brokerName = StringUtils.isNotBlank(bAckMsg.getBrokerName()) ? + bAckMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName(); + String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + brokerName; PopCheckPoint point = map.get(mergeKey); if (point == null) { if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) { From a2eafb2ad57dc6c50f8a5559c31b98b7eeeecb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Tue, 26 Nov 2024 11:37:46 +0800 Subject: [PATCH 239/265] [ISSUE #8982] Dynamically install latest Go version for e2e pipeline (#8985) * Install latest go version * Fix version error * Update pr-e2e --- .github/workflows/pr-e2e-test.yml | 5 +++-- .github/workflows/push-ci.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml index 5b4264266ef..0bc74a65ad5 100644 --- a/.github/workflows/pr-e2e-test.yml +++ b/.github/workflows/pr-e2e-test.yml @@ -188,8 +188,9 @@ jobs: test-cmd: | cd ../common && mvn -Prelease -DskipTests clean package -U cd ../rocketmq-admintools && source bin/env.sh - wget https://go.dev/dl/go1.22.6.linux-amd64.tar.gz && \ - rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.6.linux-amd64.tar.gz + LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1') + wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v job-id: 0 - name: Publish Test Report diff --git a/.github/workflows/push-ci.yml b/.github/workflows/push-ci.yml index 9e13794b318..6de8595724f 100644 --- a/.github/workflows/push-ci.yml +++ b/.github/workflows/push-ci.yml @@ -227,8 +227,9 @@ jobs: test-cmd: | cd ../common && mvn -Prelease -DskipTests clean package -U cd ../rocketmq-admintools && source bin/env.sh - wget https://go.dev/dl/go1.22.6.linux-amd64.tar.gz && \ - rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.6.linux-amd64.tar.gz + LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1') + wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \ + rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m -v job-id: 0 - name: Publish Test Report From fc2283008b82d7b06d7d6054121ea6a074ebea9f Mon Sep 17 00:00:00 2001 From: lizhimins <707364882@qq.com> Date: Tue, 26 Nov 2024 14:16:38 +0800 Subject: [PATCH 240/265] [ISSUE #8976] Modify file segment construct method (#8977) --- .../tieredstore/file/FlatFileFactory.java | 13 ++++- .../tieredstore/file/FlatFileStore.java | 2 +- .../tieredstore/provider/FileSegment.java | 7 ++- .../provider/FileSegmentFactory.java | 11 +++- .../provider/MemoryFileSegment.java | 5 +- .../provider/PosixFileSegment.java | 58 +++++++++---------- .../tieredstore/index/IndexStoreFileTest.java | 5 +- .../provider/FileSegmentFactoryTest.java | 8 ++- .../tieredstore/provider/FileSegmentTest.java | 26 ++++----- .../provider/MemoryFileSegmentTest.java | 3 +- 10 files changed, 82 insertions(+), 56 deletions(-) diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java index ccaa58e4c22..d14ea7ffdb2 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileFactory.java @@ -17,7 +17,9 @@ package org.apache.rocketmq.tieredstore.file; +import com.google.common.annotations.VisibleForTesting; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.metadata.MetadataStore; import org.apache.rocketmq.tieredstore.provider.FileSegmentFactory; @@ -28,10 +30,19 @@ public class FlatFileFactory { private final MessageStoreConfig storeConfig; private final FileSegmentFactory fileSegmentFactory; + @VisibleForTesting public FlatFileFactory(MetadataStore metadataStore, MessageStoreConfig storeConfig) { this.metadataStore = metadataStore; this.storeConfig = storeConfig; - this.fileSegmentFactory = new FileSegmentFactory(metadataStore, storeConfig); + this.fileSegmentFactory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor()); + } + + public FlatFileFactory(MetadataStore metadataStore, + MessageStoreConfig storeConfig, MessageStoreExecutor executor) { + + this.metadataStore = metadataStore; + this.storeConfig = storeConfig; + this.fileSegmentFactory = new FileSegmentFactory(metadataStore, storeConfig, executor); } public MessageStoreConfig getStoreConfig() { diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java index 70ba2178010..700f12d0d6e 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/FlatFileStore.java @@ -50,7 +50,7 @@ public FlatFileStore(MessageStoreConfig storeConfig, MetadataStore metadataStore this.storeConfig = storeConfig; this.metadataStore = metadataStore; this.executor = executor; - this.flatFileFactory = new FlatFileFactory(metadataStore, storeConfig); + this.flatFileFactory = new FlatFileFactory(metadataStore, storeConfig, executor); this.flatFileConcurrentMap = new ConcurrentHashMap<>(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java index f60fc95d23e..1140add67d1 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegment.java @@ -23,6 +23,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.locks.ReentrantLock; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.AppendResult; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.exception.TieredStoreErrorCode; @@ -45,6 +46,7 @@ public abstract class FileSegment implements Comparable, FileSegmen protected final MessageStoreConfig storeConfig; protected final long maxSize; + protected final MessageStoreExecutor executor; protected final ReentrantLock fileLock = new ReentrantLock(); protected final Semaphore commitLock = new Semaphore(1); @@ -58,13 +60,14 @@ public abstract class FileSegment implements Comparable, FileSegmen protected volatile FileSegmentInputStream fileSegmentInputStream; protected volatile CompletableFuture flightCommitRequest; - public FileSegment(MessageStoreConfig storeConfig, - FileSegmentType fileType, String filePath, long baseOffset) { + public FileSegment(MessageStoreConfig storeConfig, FileSegmentType fileType, + String filePath, long baseOffset, MessageStoreExecutor executor) { this.storeConfig = storeConfig; this.fileType = fileType; this.filePath = filePath; this.baseOffset = baseOffset; + this.executor = executor; this.maxSize = this.getMaxSizeByFileType(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java index 5146d46dbc1..ace6d8f08fd 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactory.java @@ -19,6 +19,7 @@ import java.lang.reflect.Constructor; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.metadata.MetadataStore; @@ -26,16 +27,20 @@ public class FileSegmentFactory { private final MetadataStore metadataStore; private final MessageStoreConfig storeConfig; + private final MessageStoreExecutor executor; private final Constructor fileSegmentConstructor; - public FileSegmentFactory(MetadataStore metadataStore, MessageStoreConfig storeConfig) { + public FileSegmentFactory(MetadataStore metadataStore, + MessageStoreConfig storeConfig, MessageStoreExecutor executor) { + try { this.storeConfig = storeConfig; this.metadataStore = metadataStore; + this.executor = executor; Class clazz = Class.forName(storeConfig.getTieredBackendServiceProvider()).asSubclass(FileSegment.class); fileSegmentConstructor = clazz.getConstructor( - MessageStoreConfig.class, FileSegmentType.class, String.class, Long.TYPE); + MessageStoreConfig.class, FileSegmentType.class, String.class, Long.TYPE, MessageStoreExecutor.class); } catch (Exception e) { throw new RuntimeException(e); } @@ -51,7 +56,7 @@ public MessageStoreConfig getStoreConfig() { public FileSegment createSegment(FileSegmentType fileType, String filePath, long baseOffset) { try { - return fileSegmentConstructor.newInstance(this.storeConfig, fileType, filePath, baseOffset); + return fileSegmentConstructor.newInstance(this.storeConfig, fileType, filePath, baseOffset, executor); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java index b3f10113939..93ad74541b6 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegment.java @@ -19,6 +19,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream; import org.apache.rocketmq.tieredstore.util.MessageStoreUtil; @@ -35,9 +36,9 @@ public class MemoryFileSegment extends FileSegment { protected boolean checkSize = true; public MemoryFileSegment(MessageStoreConfig storeConfig, - FileSegmentType fileType, String filePath, long baseOffset) { + FileSegmentType fileType, String filePath, long baseOffset, MessageStoreExecutor executor) { - super(storeConfig, fileType, filePath, baseOffset); + super(storeConfig, fileType, filePath, baseOffset, executor); memStore = ByteBuffer.allocate(10000); memStore.position((int) getSize()); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java index 656af2ba1c6..3ab5914161d 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/PosixFileSegment.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.tieredstore.provider; import com.google.common.base.Stopwatch; +import com.google.common.base.Supplier; import com.google.common.io.ByteStreams; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; @@ -30,6 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream; @@ -58,9 +60,9 @@ public class PosixFileSegment extends FileSegment { private volatile FileChannel writeFileChannel; public PosixFileSegment(MessageStoreConfig storeConfig, - FileSegmentType fileType, String filePath, long baseOffset) { + FileSegmentType fileType, String filePath, long baseOffset, MessageStoreExecutor executor) { - super(storeConfig, fileType, filePath, baseOffset); + super(storeConfig, fileType, filePath, baseOffset, executor); // basePath String basePath = StringUtils.defaultString(storeConfig.getTieredStoreFilePath(), @@ -168,32 +170,30 @@ public CompletableFuture read0(long position, int length) { AttributesBuilder attributesBuilder = newAttributesBuilder() .put(LABEL_OPERATION, OPERATION_POSIX_READ); - CompletableFuture future = new CompletableFuture<>(); - ByteBuffer byteBuffer = ByteBuffer.allocate(length); - try { - readFileChannel.position(position); - readFileChannel.read(byteBuffer); - byteBuffer.flip(); - byteBuffer.limit(length); - - attributesBuilder.put(LABEL_SUCCESS, true); - long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); - TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build()); - - Attributes metricsAttributes = newAttributesBuilder() - .put(LABEL_OPERATION, OPERATION_POSIX_READ) - .build(); - int downloadedBytes = byteBuffer.remaining(); - TieredStoreMetricsManager.downloadBytes.record(downloadedBytes, metricsAttributes); - - future.complete(byteBuffer); - } catch (IOException e) { - long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); - attributesBuilder.put(LABEL_SUCCESS, false); - TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build()); - future.completeExceptionally(e); - } - return future; + return CompletableFuture.supplyAsync((Supplier) () -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(length); + try { + readFileChannel.position(position); + readFileChannel.read(byteBuffer); + byteBuffer.flip(); + byteBuffer.limit(length); + + attributesBuilder.put(LABEL_SUCCESS, true); + long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); + TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build()); + + Attributes metricsAttributes = newAttributesBuilder() + .put(LABEL_OPERATION, OPERATION_POSIX_READ) + .build(); + int downloadedBytes = byteBuffer.remaining(); + TieredStoreMetricsManager.downloadBytes.record(downloadedBytes, metricsAttributes); + } catch (IOException e) { + long costTime = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); + attributesBuilder.put(LABEL_SUCCESS, false); + TieredStoreMetricsManager.providerRpcLatency.record(costTime, attributesBuilder.build()); + } + return byteBuffer; + }, executor.bufferFetchExecutor); } @Override @@ -229,6 +229,6 @@ public CompletableFuture commit0( return false; } return true; - }); + }, executor.bufferCommitExecutor); } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java index 48bf9ba4c74..d19b562463d 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java @@ -29,6 +29,7 @@ import java.util.concurrent.Executors; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.AppendResult; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.provider.FileSegment; @@ -219,7 +220,7 @@ public void doCompactionTest() { ByteBuffer byteBuffer = indexStoreFile.doCompaction(); FileSegment fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.INDEX, filePath, 0L); + storeConfig, FileSegmentType.INDEX, filePath, 0L, new MessageStoreExecutor()); fileSegment.append(byteBuffer, timestamp); fileSegment.commitAsync().join(); Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize()); @@ -252,7 +253,7 @@ public void queryAsyncFromSegmentFileTest() throws ExecutionException, Interrupt ByteBuffer byteBuffer = indexStoreFile.doCompaction(); FileSegment fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.INDEX, filePath, 0L); + storeConfig, FileSegmentType.INDEX, filePath, 0L, new MessageStoreExecutor()); fileSegment.append(byteBuffer, timestamp); fileSegment.commitAsync().join(); Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize()); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java index 1efbc3f9ee3..3b44b10f47b 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentFactoryTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.tieredstore.provider; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.metadata.DefaultMetadataStore; import org.apache.rocketmq.tieredstore.metadata.MetadataStore; @@ -34,9 +35,10 @@ public void fileSegmentInstanceTest() throws ClassNotFoundException, NoSuchMetho MessageStoreConfig storeConfig = new MessageStoreConfig(); storeConfig.setTieredStoreCommitLogMaxSize(1024); storeConfig.setTieredStoreFilePath(storePath); + MessageStoreExecutor executor = new MessageStoreExecutor(); MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, executor); Assert.assertEquals(metadataStore, factory.getMetadataStore()); Assert.assertEquals(storeConfig, factory.getStoreConfig()); @@ -60,7 +62,9 @@ public void fileSegmentInstanceTest() throws ClassNotFoundException, NoSuchMetho () -> factory.createSegment(null, null, 0L)); storeConfig.setTieredBackendServiceProvider(null); Assert.assertThrows(RuntimeException.class, - () -> new FileSegmentFactory(metadataStore, storeConfig)); + () -> new FileSegmentFactory(metadataStore, storeConfig, executor)); + + executor.shutdown(); MessageStoreUtilTest.deleteStoreDirectory(storePath); } } diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java index 2bba3d01370..26844113cd0 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/FileSegmentTest.java @@ -74,7 +74,7 @@ public void shutdown() { public void fileAttributesTest() { int baseOffset = 1000; FileSegment fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor); // for default value check Assert.assertEquals(baseOffset, fileSegment.getBaseOffset()); @@ -104,9 +104,9 @@ public void fileAttributesTest() { @Test public void fileSortByOffsetTest() { FileSegment fileSegment1 = new PosixFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 200L); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 200L, storeExecutor); FileSegment fileSegment2 = new PosixFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor); FileSegment[] fileSegments = new FileSegment[] {fileSegment1, fileSegment2}; Arrays.sort(fileSegments); Assert.assertEquals(fileSegments[0], fileSegment2); @@ -116,17 +116,17 @@ public void fileSortByOffsetTest() { @Test public void fileMaxSizeTest() { FileSegment fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor); Assert.assertEquals(storeConfig.getTieredStoreCommitLogMaxSize(), fileSegment.getMaxSize()); fileSegment.destroyFile(); fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.CONSUME_QUEUE, MessageStoreUtil.toFilePath(mq), 100L); + storeConfig, FileSegmentType.CONSUME_QUEUE, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor); Assert.assertEquals(storeConfig.getTieredStoreConsumeQueueMaxSize(), fileSegment.getMaxSize()); fileSegment.destroyFile(); fileSegment = new PosixFileSegment( - storeConfig, FileSegmentType.INDEX, MessageStoreUtil.toFilePath(mq), 100L); + storeConfig, FileSegmentType.INDEX, MessageStoreUtil.toFilePath(mq), 100L, storeExecutor); Assert.assertEquals(Long.MAX_VALUE, fileSegment.getMaxSize()); fileSegment.destroyFile(); } @@ -134,7 +134,7 @@ public void fileMaxSizeTest() { @Test public void unexpectedCaseTest() { MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor()); FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset); fileSegment.initPosition(fileSegment.getSize()); @@ -157,7 +157,7 @@ public void unexpectedCaseTest() { @Test public void commitLogTest() throws InterruptedException { MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor()); FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset); long lastSize = fileSegment.getSize(); fileSegment.initPosition(fileSegment.getSize()); @@ -225,7 +225,7 @@ public void commitLogTest() throws InterruptedException { @Test public void consumeQueueTest() throws ClassNotFoundException, NoSuchMethodException { MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor()); FileSegment fileSegment = factory.createConsumeQueueFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset); long storeTimestamp = System.currentTimeMillis(); @@ -258,7 +258,7 @@ public void consumeQueueTest() throws ClassNotFoundException, NoSuchMethodExcept @Test public void fileSegmentReadTest() throws ClassNotFoundException, NoSuchMethodException { MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, new MessageStoreExecutor()); FileSegment fileSegment = factory.createConsumeQueueFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset); long storeTimestamp = System.currentTimeMillis(); @@ -292,7 +292,7 @@ public void fileSegmentReadTest() throws ClassNotFoundException, NoSuchMethodExc @Test public void commitFailedThenSuccessTest() { MemoryFileSegment segment = new MemoryFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor); long lastSize = segment.getSize(); segment.setCheckSize(false); @@ -352,7 +352,7 @@ public void commitFailedThenSuccessTest() { public void commitFailedMoreTimes() { long startTime = System.currentTimeMillis(); MemoryFileSegment segment = new MemoryFileSegment( - storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset); + storeConfig, FileSegmentType.COMMIT_LOG, MessageStoreUtil.toFilePath(mq), baseOffset, storeExecutor); long lastSize = segment.getSize(); segment.setCheckSize(false); @@ -419,7 +419,7 @@ public void commitFailedMoreTimes() { @Test public void handleCommitExceptionTest() { MetadataStore metadataStore = new DefaultMetadataStore(storeConfig); - FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig); + FileSegmentFactory factory = new FileSegmentFactory(metadataStore, storeConfig, storeExecutor); { FileSegment fileSegment = factory.createCommitLogFileSegment(MessageStoreUtil.toFilePath(mq), baseOffset); diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java index cc9793dc886..72396506b10 100644 --- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java +++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MemoryFileSegmentTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.tieredstore.MessageStoreConfig; +import org.apache.rocketmq.tieredstore.MessageStoreExecutor; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.stream.FileSegmentInputStream; import org.apache.rocketmq.tieredstore.util.MessageStoreUtil; @@ -34,7 +35,7 @@ public class MemoryFileSegmentTest { public void memoryTest() throws IOException { MemoryFileSegment fileSegment = new MemoryFileSegment( new MessageStoreConfig(), FileSegmentType.COMMIT_LOG, - MessageStoreUtil.toFilePath(new MessageQueue()), 0L); + MessageStoreUtil.toFilePath(new MessageQueue()), 0L, new MessageStoreExecutor()); Assert.assertFalse(fileSegment.exists()); fileSegment.createFile(); MemoryFileSegment fileSpySegment = Mockito.spy(fileSegment); From cd5071bd05acf4eae19affa5216638e28540cf0a Mon Sep 17 00:00:00 2001 From: yx9o Date: Wed, 27 Nov 2024 14:43:45 +0800 Subject: [PATCH 241/265] [ISSUE #8963] Fix code36 request sent to ns (#8964) * [ISSUE #8963] Fix code36 request sent to ns * Update DefaultMQPullConsumerImpl.java --- .../client/impl/consumer/DefaultMQPullConsumerImpl.java | 4 ++++ .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index e05c614c6d2..371a4a0dbdb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -649,6 +649,10 @@ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerN String brokerAddr = (null != destBrokerName) ? this.mQClientFactory.findBrokerAddressInPublish(destBrokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + if (UtilAll.isBlank(brokerAddr)) { + throw new MQClientException("Broker[" + destBrokerName + "] master node does not exist", null); + } + if (UtilAll.isBlank(consumerGroup)) { consumerGroup = this.defaultMQPullConsumer.getConsumerGroup(); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 4eccba8e8d4..46715cea950 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -767,6 +767,9 @@ private void sendMessageBack(MessageExt msg, int delayLevel, final String broker } else { String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + if (UtilAll.isBlank(brokerAddr)) { + throw new MQClientException("Broker[" + brokerName + "] master node does not exist", null); + } this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, brokerName, msg, this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); } From 3ae0139aa0cd75fe52d583e69f0c974d5ce2639c Mon Sep 17 00:00:00 2001 From: rongtong Date: Wed, 27 Nov 2024 16:19:06 +0800 Subject: [PATCH 242/265] [ISSUE #8968] Introduce the clearRetryTopicWhenDeleteTopic option to enable precise external deletion of topics (#8969) * Add the clearRetryTopicWhenDeleteTopic option to allow precise deletion of topics externally without the need to traverse consumerOffset * Fix check style --- .../rocketmq/broker/BrokerController.java | 7 +- .../processor/AdminBrokerProcessor.java | 71 +++++++++++-------- .../apache/rocketmq/common/BrokerConfig.java | 9 +++ 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index b907489bbfb..99e5b85d2e4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -191,7 +191,7 @@ public class BrokerController { private final NettyClientConfig nettyClientConfig; protected final MessageStoreConfig messageStoreConfig; private final AuthConfig authConfig; - protected final ConsumerOffsetManager consumerOffsetManager; + protected ConsumerOffsetManager consumerOffsetManager; protected final BroadcastOffsetManager broadcastOffsetManager; protected final ConsumerManager consumerManager; protected final ConsumerFilterManager consumerFilterManager; @@ -1313,6 +1313,11 @@ public ConsumerOffsetManager getConsumerOffsetManager() { return consumerOffsetManager; } + public void setConsumerOffsetManager(ConsumerOffsetManager consumerOffsetManager) { + this.consumerOffsetManager = consumerOffsetManager; + } + + public BroadcastOffsetManager getBroadcastOffsetManager() { return broadcastOffsetManager; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index ac882e94ab0..cc70e69a467 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -490,6 +490,7 @@ private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, R response.setBody(JSON.toJSONBytes(result)); return response; } + @Override public boolean rejectRequest() { return false; @@ -559,18 +560,17 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark(e.getMessage()); return response; - } - finally { + } finally { executionTime = System.currentTimeMillis() - startTime; InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? - InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; Attributes attributes = BrokerMetricsManager.newAttributesBuilder() - .put(LABEL_INVOCATION_STATUS, status.getName()) - .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic)) - .build(); + .put(LABEL_INVOCATION_STATUS, status.getName()) + .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic)) + .build(); BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes); } - LOGGER.info("executionTime of create topic:{} is {} ms" , topic, executionTime); + LOGGER.info("executionTime of create topic:{} is {} ms", topic, executionTime); return response; } @@ -637,8 +637,7 @@ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerCont response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark(e.getMessage()); return response; - } - finally { + } finally { executionTime = System.currentTimeMillis() - startTime; InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? InvocationStatus.SUCCESS : InvocationStatus.FAILURE; @@ -648,7 +647,7 @@ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerCont .build(); BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes); } - LOGGER.info("executionTime of all topics:{} is {} ms" , topicNames, executionTime); + LOGGER.info("executionTime of all topics:{} is {} ms", topicNames, executionTime); return response; } @@ -725,21 +724,28 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, } } - final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); - // delete pop retry topics first - try { + List topicsToClean = new ArrayList<>(); + topicsToClean.add(topic); + + if (brokerController.getBrokerConfig().isClearRetryTopicWhenDeleteTopic()) { + final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); for (String group : groups) { final String popRetryTopicV2 = KeyBuilder.buildPopRetryTopic(topic, group, true); if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV2) != null) { - deleteTopicInBroker(popRetryTopicV2); + topicsToClean.add(popRetryTopicV2); } final String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group); if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV1) != null) { - deleteTopicInBroker(popRetryTopicV1); + topicsToClean.add(popRetryTopicV1); } } - // delete topic - deleteTopicInBroker(topic); + } + + try { + for (String topicToClean : topicsToClean) { + // delete topic + deleteTopicInBroker(topicToClean); + } } catch (Throwable t) { return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); } @@ -982,10 +988,10 @@ private synchronized RemotingCommand updateColdDataFlowCtrGroupConfig(ChannelHan String consumerGroup = String.valueOf(key); Long threshold = Long.valueOf(String.valueOf(value)); this.brokerController.getColdDataCgCtrService() - .addOrUpdateGroupConfig(consumerGroup, threshold); + .addOrUpdateGroupConfig(consumerGroup, threshold); } catch (Exception e) { LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}", - key, value, e); + key, value, e); } }); } else { @@ -1598,12 +1604,12 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c response.setCode(ResponseCode.SUCCESS); response.setRemark(null); long executionTime = System.currentTimeMillis() - startTime; - LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms" ,config.getGroupName() ,executionTime); + LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms", config.getGroupName(), executionTime); InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ? - InvocationStatus.SUCCESS : InvocationStatus.FAILURE; + InvocationStatus.SUCCESS : InvocationStatus.FAILURE; Attributes attributes = BrokerMetricsManager.newAttributesBuilder() - .put(LABEL_INVOCATION_STATUS, status.getName()) - .build(); + .put(LABEL_INVOCATION_STATUS, status.getName()) + .build(); BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes); return response; } @@ -2083,13 +2089,13 @@ private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) /** * Reset consumer offset. * - * @param topic Required, not null. - * @param group Required, not null. - * @param queueId if target queue ID is negative, all message queues will be reset; otherwise, only the target queue - * would get reset. + * @param topic Required, not null. + * @param group Required, not null. + * @param queueId if target queue ID is negative, all message queues will be reset; otherwise, only the target queue + * would get reset. * @param timestamp if timestamp is negative, offset would be reset to broker offset at the time being; otherwise, - * binary search is performed to locate target offset. - * @param offset Target offset to reset to if target queue ID is properly provided. + * binary search is performed to locate target offset. + * @param offset Target offset to reset to if target queue ID is properly provided. * @return Affected queues and their new offset */ private RemotingCommand resetOffsetInner(String topic, String group, int queueId, long timestamp, Long offset) { @@ -3371,7 +3377,8 @@ private boolean validateBlackListConfigExist(Properties properties) { return false; } - private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class); String requestTopic = requestHeader.getTopic(); MessageStore messageStore = brokerController.getMessageStore(); @@ -3428,7 +3435,9 @@ private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerCo return result; } - private boolean processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean printDetail, long checkpointByStoreTime) { + private boolean processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic, + RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean printDetail, + long checkpointByStoreTime) { boolean processResult = true; for (Map.Entry queueEntry : queueMap.entrySet()) { Integer queueId = queueEntry.getKey(); diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 9d8d9135217..c0b557dfa11 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -435,6 +435,7 @@ public class BrokerConfig extends BrokerIdentity { private boolean appendCkAsync = false; + private boolean clearRetryTopicWhenDeleteTopic = true; private boolean enableLmqStats = false; @@ -1908,6 +1909,14 @@ public void setAppendCkAsync(boolean appendCkAsync) { this.appendCkAsync = appendCkAsync; } + public boolean isClearRetryTopicWhenDeleteTopic() { + return clearRetryTopicWhenDeleteTopic; + } + + public void setClearRetryTopicWhenDeleteTopic(boolean clearRetryTopicWhenDeleteTopic) { + this.clearRetryTopicWhenDeleteTopic = clearRetryTopicWhenDeleteTopic; + } + public boolean isEnableLmqStats() { return enableLmqStats; } From adc1c748ef57c2e31ed23334193a8bd6d9c6cf84 Mon Sep 17 00:00:00 2001 From: weihubeats Date: Thu, 28 Nov 2024 10:31:23 +0800 Subject: [PATCH 243/265] [ISSUE #8991] prepareHeartbeatData should not be set by default subscriptionDataSet data (#8992) * Adding null does not update * rolling back * :bugfix: prepareHeartbeatData should not be set by default subscriptionDataSet data --- .../apache/rocketmq/client/impl/factory/MQClientInstance.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index ad0676d091c..8cc910487c1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -869,7 +869,6 @@ private HeartbeatData prepareHeartbeatData(boolean isWithoutSub) { consumerData.setConsumeType(impl.consumeType()); consumerData.setMessageModel(impl.messageModel()); consumerData.setConsumeFromWhere(impl.consumeFromWhere()); - consumerData.getSubscriptionDataSet().addAll(impl.subscriptions()); consumerData.setUnitMode(impl.isUnitMode()); if (!isWithoutSub) { consumerData.getSubscriptionDataSet().addAll(impl.subscriptions()); From d2c436b7fb86eec9d9abe89766f8b4a60cbb721f Mon Sep 17 00:00:00 2001 From: weihubeats Date: Thu, 28 Nov 2024 18:01:45 +0800 Subject: [PATCH 244/265] [ISSUE #7199] grpcClientChannel header add null judgement (#7238) adding a null judgement --- .../rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java index 714d0bf019e..f05251c58c5 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java @@ -30,6 +30,7 @@ import io.grpc.stub.StreamObserver; import io.netty.channel.Channel; import io.netty.channel.ChannelId; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.common.constant.LoggerName; @@ -210,7 +211,7 @@ protected CompletableFuture processCheckTransaction(CheckTransactionStateR protected CompletableFuture processGetConsumerRunningInfo(RemotingCommand command, GetConsumerRunningInfoRequestHeader header, CompletableFuture> responseFuture) { - if (!header.isJstackEnable()) { + if (Objects.isNull(header) || !header.isJstackEnable()) { return CompletableFuture.completedFuture(null); } this.writeTelemetryCommand(TelemetryCommand.newBuilder() From 804847e87765f835afa147887f9507b8a41ae08c Mon Sep 17 00:00:00 2001 From: qianye Date: Fri, 29 Nov 2024 17:18:49 +0800 Subject: [PATCH 245/265] test (#9010) --- .../consumer/DefaultMQPullConsumer.java | 29 +++++++++++++++++++ .../consumer/DefaultMQPullConsumerImpl.java | 9 ++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 9e7a86d9b49..38841e41287 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -16,9 +16,11 @@ */ package org.apache.rocketmq.client.consumer; +import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; @@ -33,7 +35,9 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.NamespaceUtil; +import org.apache.rocketmq.remoting.protocol.filter.FilterAPI; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; /** * @deprecated Default pulling consumer. This class will be removed in 2022, and a better implementation @@ -77,6 +81,8 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume * Topic set you want to register */ private Set registerTopics = new HashSet<>(); + + private final Set registerSubscriptions = Collections.newSetFromMap(new ConcurrentHashMap<>()); /** * Queue allocation algorithm */ @@ -255,6 +261,29 @@ public void setRegisterTopics(Set registerTopics) { this.registerTopics = withNamespace(registerTopics); } + public Set getRegisterSubscriptions() { + return registerSubscriptions; + } + + public void addRegisterSubscriptions(String topic, MessageSelector messageSelector) throws MQClientException { + try { + if (messageSelector == null) { + messageSelector = MessageSelector.byTag(SubscriptionData.SUB_ALL); + } + + SubscriptionData subscriptionData = FilterAPI.build(withNamespace(topic), + messageSelector.getExpression(), messageSelector.getExpressionType()); + + this.registerSubscriptions.add(subscriptionData); + } catch (Exception e) { + throw new MQClientException("add subscription exception", e); + } + } + + public void clearRegisterSubscriptions() { + this.registerSubscriptions.clear(); + } + /** * This method will be removed or it's visibility will be changed in a certain version after April 5, 2020, so * please do not use this method. diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 371a4a0dbdb..9d46e28f5d4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -54,6 +54,8 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -63,8 +65,6 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; /** * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumerImpl} is recommend to use @@ -356,6 +356,11 @@ public ConsumeFromWhere consumeFromWhere() { @Override public Set subscriptions() { + Set registerSubscriptions = defaultMQPullConsumer.getRegisterSubscriptions(); + if (registerSubscriptions != null && !registerSubscriptions.isEmpty()) { + return registerSubscriptions; + } + Set result = new HashSet<>(); Set topics = this.defaultMQPullConsumer.getRegisterTopics(); From fb3b87da1bb3337039cc80d7a3fcf2dff4bd6ce3 Mon Sep 17 00:00:00 2001 From: Liu Shengzhong Date: Mon, 2 Dec 2024 10:01:51 +0800 Subject: [PATCH 246/265] [ISSUE #8984] Fix the broker switch enableMixedMessageType doesn't work --- .../processor/AdminBrokerProcessor.java | 29 ++++++++----- .../processor/AdminBrokerProcessorTest.java | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index cc70e69a467..fc3b6182731 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -76,6 +76,7 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.common.TopicAttributes; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UnlockCallback; import org.apache.rocketmq.common.UtilAll; @@ -534,11 +535,15 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext String attributesModification = requestHeader.getAttributes(); topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); - if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED - && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("MIXED message type is not supported."); - return response; + if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) { + // Get attribute by key with prefix sign + String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(); + String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey); + if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("MIXED message type is not supported."); + return response; + } } if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) { @@ -609,11 +614,15 @@ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerCont return response; } } - if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED - && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("MIXED message type is not supported."); - return response; + if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) { + // Get attribute by key with prefix sign + String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(); + String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey); + if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("MIXED message type is not supported."); + return response; + } } if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) { LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}", diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index d87f5133552..48ddb891728 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.attribute.AttributeParser; import org.apache.rocketmq.common.constant.FIleReadaheadMode; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -330,6 +331,19 @@ public void testUpdateAndCreateTopic() throws Exception { request = buildCreateTopicRequest(topic); response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + // test deny MIXED topic type + brokerController.getBrokerConfig().setEnableMixedMessageType(false); + topic = "TEST_MIXED_TYPE"; + Map attributes = new HashMap<>(); + attributes.put("+message.type", "MIXED"); + request = buildCreateTopicRequest(topic, attributes); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + // test allow MIXED topic type + brokerController.getBrokerConfig().setEnableMixedMessageType(true); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } @Test @@ -355,6 +369,20 @@ public void testUpdateAndCreateTopicList() throws RemotingCommandException { //test no changes response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + // test deny MIXED topic type + brokerController.getBrokerConfig().setEnableMixedMessageType(false); + topicList.add("TEST_MIXED_TYPE"); + topicList.add("TEST_MIXED_TYPE1"); + Map attributes = new HashMap<>(); + attributes.put("+message.type", "MIXED"); + request = buildCreateTopicListRequest(topicList, attributes); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + // test allow MIXED topic type + brokerController.getBrokerConfig().setEnableMixedMessageType(true); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } @Test @@ -1312,18 +1340,29 @@ private ResetOffsetRequestHeader createRequestHeader(String topic,String group,l } private RemotingCommand buildCreateTopicRequest(String topic) { + return buildCreateTopicRequest(topic, null); + } + + private RemotingCommand buildCreateTopicRequest(String topic, Map attributes) { CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); requestHeader.setTopic(topic); requestHeader.setTopicFilterType(TopicFilterType.SINGLE_TAG.name()); requestHeader.setReadQueueNums(8); requestHeader.setWriteQueueNums(8); requestHeader.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); + if (attributes != null) { + requestHeader.setAttributes(AttributeParser.parseToString(attributes)); + } RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); request.makeCustomHeaderToNet(); return request; } private RemotingCommand buildCreateTopicListRequest(List topicList) { + return buildCreateTopicListRequest(topicList, null); + } + + private RemotingCommand buildCreateTopicListRequest(List topicList, Map attributes) { List topicConfigList = new ArrayList<>(); for (String topic:topicList) { TopicConfig topicConfig = new TopicConfig(topic); @@ -1333,6 +1372,9 @@ private RemotingCommand buildCreateTopicListRequest(List topicList) { topicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); topicConfig.setTopicSysFlag(0); topicConfig.setOrder(false); + if (attributes != null) { + topicConfig.setAttributes(new HashMap<>(attributes)); + } topicConfigList.add(topicConfig); } RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC_LIST, null); From 93d7e80a972ec07f2fa853355449aba8060eae12 Mon Sep 17 00:00:00 2001 From: jiao jianan <81030751+jjastan@users.noreply.github.com> Date: Mon, 2 Dec 2024 10:03:16 +0800 Subject: [PATCH 247/265] [ISSUE #8950] Remove Redundant nullcheck of configPath (#8951) Co-authored-by: jiaoja --- .../rocketmq/container/BrokerContainerProcessor.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java index 5ced0825761..80dd6ccb13b 100644 --- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java +++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java @@ -86,7 +86,7 @@ public boolean rejectRequest() { } private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx, - RemotingCommand request) throws Exception { + RemotingCommand request) throws Exception { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final AddBrokerRequestHeader requestHeader = (AddBrokerRequestHeader) request.decodeCommandCustomHeader(AddBrokerRequestHeader.class); @@ -124,10 +124,7 @@ private synchronized RemotingCommand addBroker(ChannelHandlerContext ctx, MixAll.properties2Object(brokerProperties, messageStoreConfig); messageStoreConfig.setHaListenPort(brokerConfig.getListenPort() + 1); - - if (configPath != null && !configPath.isEmpty()) { - brokerConfig.setBrokerConfigPath(configPath); - } + brokerConfig.setBrokerConfigPath(configPath); if (!messageStoreConfig.isEnableDLegerCommitLog()) { if (!brokerConfig.isEnableControllerMode()) { From ea1228a30ad4d12259a653585eb73ec12e3374b1 Mon Sep 17 00:00:00 2001 From: Humkum <1109939087@qq.com> Date: Mon, 2 Dec 2024 10:04:17 +0800 Subject: [PATCH 248/265] [ISSUE #8966] Feat: add remote address information to acl perm error --- .../rocketmq/remoting/netty/NettyRemotingAbstract.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index b0c7099b9dc..3d4e62f9430 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -320,9 +320,10 @@ private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingC return () -> { Exception exception = null; RemotingCommand response; + String remoteAddr = null; try { - String remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + remoteAddr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); try { doBeforeRpcHooks(remoteAddr, cmd); } catch (AbortProcessException e) { @@ -359,7 +360,7 @@ private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingC response.setOpaque(opaque); writeResponse(ctx.channel(), cmd, response); } catch (Throwable e) { - log.error("process request exception", e); + log.error("process request exception, remoteAddr: {}", remoteAddr, e); log.error(cmd.toString()); if (!cmd.isOnewayRPC()) { From 9a891f1d49af0cfe4e384bc28a0bf0eeed02587f Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Mon, 2 Dec 2024 10:07:54 +0800 Subject: [PATCH 249/265] fix: multiple patches during long running tests for LMQ over RocksDB (#8915) * fix: multiple patches during long running tests for LMQ over RocksDB Signed-off-by: Li Zhanhui * fix: fix a bug in RocksGroupCommitService; remove RocksDBConsumeQueueStore#findConsumeQueueMap override Signed-off-by: Li Zhanhui * fix: async fsync on RocksDB WAL flush Signed-off-by: Li Zhanhui * fix: use a dedicated thread to flush and sync RocksDB WAL Signed-off-by: Li Zhanhui * fix: trigger WAL rolling according to estimated WAL file size Signed-off-by: Li Zhanhui * chore: add doc, explaining config RocksDB instance flush/sync strategy Signed-off-by: Li Zhanhui * fix: data-version should be per table Signed-off-by: Li Zhanhui * fix: test case: RocksdbTransferOffsetAndCqTest Signed-off-by: Li Zhanhui --------- Signed-off-by: Li Zhanhui --- .../rocketmq/broker/BrokerController.java | 2 +- .../broker/config/v2/ConfigHelper.java | 4 +- .../broker/config/v2/ConfigStorage.java | 156 +++++++++++++++++- .../config/v2/ConsumerOffsetManagerV2.java | 8 +- .../config/v2/SubscriptionGroupManagerV2.java | 6 +- .../config/v2/TopicConfigManagerV2.java | 6 +- .../v2/ConsumerOffsetManagerV2Test.java | 19 ++- .../v2/SubscriptionGroupManagerV2Test.java | 15 +- .../config/v2/TopicConfigManagerV2Test.java | 25 ++- .../RocksdbTransferOffsetAndCqTest.java | 16 +- .../common/config/AbstractRocksDBStorage.java | 34 ++-- .../rocketmq/common/config/ConfigHelper.java | 32 ++-- .../common/config/ConfigRocksDBStorage.java | 2 +- .../org/apache/rocketmq/store/CommitLog.java | 16 +- .../store/config/MessageStoreConfig.java | 24 +++ .../store/queue/RocksDBConsumeQueueStore.java | 95 +++++++---- .../store/queue/RocksGroupCommitService.java | 103 ++++++++++++ .../store/rocksdb/RocksDBOptionsFactory.java | 7 +- 18 files changed, 474 insertions(+), 96 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 99e5b85d2e4..e1edd2f5126 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -344,7 +344,7 @@ public BrokerController( this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); this.broadcastOffsetManager = new BroadcastOffsetManager(this); if (ConfigManagerVersion.V2.getVersion().equals(brokerConfig.getConfigManagerVersion())) { - this.configStorage = new ConfigStorage(messageStoreConfig.getStorePathRootDir()); + this.configStorage = new ConfigStorage(messageStoreConfig); this.topicConfigManager = new TopicConfigManagerV2(this, configStorage); this.subscriptionGroupManager = new SubscriptionGroupManagerV2(this, configStorage); this.consumerOffsetManager = new ConsumerOffsetManagerV2(this, configStorage); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java index 8183a1f8358..29a7c313bab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigHelper.java @@ -64,7 +64,7 @@ public static Optional loadDataVersion(ConfigStorage configStorage, Tab return Optional.empty(); } - public static void stampDataVersion(WriteBatch writeBatch, DataVersion dataVersion, long stateMachineVersion) + public static void stampDataVersion(WriteBatch writeBatch, TableId table, DataVersion dataVersion, long stateMachineVersion) throws RocksDBException { // Increase data version dataVersion.nextVersion(stateMachineVersion); @@ -75,7 +75,7 @@ public static void stampDataVersion(WriteBatch writeBatch, DataVersion dataVersi ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(Long.BYTES * 3); try { keyBuf.writeByte(TablePrefix.TABLE.getValue()); - keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue()); + keyBuf.writeShort(table.getValue()); keyBuf.writeByte(RecordPrefix.DATA_VERSION.getValue()); keyBuf.writeBytes(ConfigStorage.DATA_VERSION_KEY_BYTES); valueBuf.writeLong(dataVersion.getStateVersion()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java index 6bc62957a86..c4056d142fc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConfigStorage.java @@ -16,17 +16,29 @@ */ package org.apache.rocketmq.broker.config.v2; +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.buffer.PooledByteBufAllocatorMetric; import io.netty.util.internal.PlatformDependent; import java.io.File; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.config.AbstractRocksDBStorage; import org.apache.rocketmq.common.config.ConfigHelper; +import org.apache.rocketmq.store.config.MessageStoreConfig; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.FlushOptions; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -43,8 +55,48 @@ public class ConfigStorage extends AbstractRocksDBStorage { public static final String DATA_VERSION_KEY = "data_version"; public static final byte[] DATA_VERSION_KEY_BYTES = DATA_VERSION_KEY.getBytes(StandardCharsets.UTF_8); - public ConfigStorage(String storePath) { - super(storePath + File.separator + "config" + File.separator + "rdb"); + private final ScheduledExecutorService scheduledExecutorService; + + /** + * Number of write ops since previous flush. + */ + private final AtomicInteger writeOpsCounter; + + private final AtomicLong estimateWalFileSize = new AtomicLong(0L); + + private final MessageStoreConfig messageStoreConfig; + + private final FlushSyncService flushSyncService; + + public ConfigStorage(MessageStoreConfig messageStoreConfig) { + super(messageStoreConfig.getStorePathRootDir() + File.separator + "config" + File.separator + "rdb"); + this.messageStoreConfig = messageStoreConfig; + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("config-storage-%d") + .build(); + scheduledExecutorService = new ScheduledThreadPoolExecutor(1, threadFactory); + writeOpsCounter = new AtomicInteger(0); + this.flushSyncService = new FlushSyncService(); + this.flushSyncService.setDaemon(true); + } + + private void statNettyMemory() { + PooledByteBufAllocatorMetric metric = AbstractRocksDBStorage.POOLED_ALLOCATOR.metric(); + LOGGER.info("Netty Memory Usage: {}", metric); + } + + @Override + public synchronized boolean start() { + boolean started = super.start(); + if (started) { + scheduledExecutorService.scheduleWithFixedDelay(() -> statRocksdb(LOGGER), 1, 10, TimeUnit.SECONDS); + scheduledExecutorService.scheduleWithFixedDelay(this::statNettyMemory, 10, 10, TimeUnit.SECONDS); + this.flushSyncService.start(); + } else { + LOGGER.error("Failed to start config storage"); + } + return started; } @Override @@ -58,7 +110,7 @@ protected boolean postLoad() { initOptions(); List cfDescriptors = new ArrayList<>(); - ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigOptions(); + ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions(); this.cfOptions.add(defaultOptions); cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); @@ -66,7 +118,7 @@ protected boolean postLoad() { open(cfDescriptors); this.defaultCFHandle = cfHandles.get(0); - } catch (final Exception e) { + } catch (Exception e) { AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e); return false; } @@ -75,7 +127,8 @@ protected boolean postLoad() { @Override protected void preShutdown() { - + scheduledExecutorService.shutdown(); + flushSyncService.shutdown(); } protected void initOptions() { @@ -105,6 +158,12 @@ public byte[] get(ByteBuffer key) throws RocksDBException { public void write(WriteBatch writeBatch) throws RocksDBException { db.write(ableWalWriteOptions, writeBatch); + accountWriteOps(writeBatch.getDataSize()); + } + + private void accountWriteOps(long dataSize) { + writeOpsCounter.incrementAndGet(); + estimateWalFileSize.addAndGet(dataSize); } public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) { @@ -125,4 +184,91 @@ public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) { return iterator; } } + + /** + * RocksDB writes contain 3 stages: application memory buffer --> OS Page Cache --> Disk. + * Given that we are having DBOptions::manual_wal_flush, we need to manually call DB::FlushWAL and DB::SyncWAL + * Note: DB::FlushWAL(true) will internally call DB::SyncWAL. + *

    + * See Flush And Sync WAL + */ + class FlushSyncService extends ServiceThread { + + private long lastSyncTime = 0; + + private static final long MAX_SYNC_INTERVAL_IN_MILLIS = 100; + + private final Stopwatch stopwatch = Stopwatch.createUnstarted(); + + private final FlushOptions flushOptions = new FlushOptions(); + + @Override + public String getServiceName() { + return "FlushSyncService"; + } + + @Override + public void run() { + flushOptions.setAllowWriteStall(false); + flushOptions.setWaitForFlush(true); + log.info("{} service started", this.getServiceName()); + while (!this.isStopped()) { + try { + this.waitForRunning(10); + this.flushAndSyncWAL(false); + } catch (Exception e) { + log.warn("{} service has exception. ", this.getServiceName(), e); + } + } + try { + flushAndSyncWAL(true); + } catch (Exception e) { + log.warn("{} raised an exception while performing flush-and-sync WAL on exit", + this.getServiceName(), e); + } + flushOptions.close(); + log.info("{} service end", this.getServiceName()); + } + + private void flushAndSyncWAL(boolean onExit) throws RocksDBException { + int writeOps = writeOpsCounter.get(); + if (0 == writeOps) { + // No write ops to flush + return; + } + + /* + * Normally, when MemTables become full then immutable, RocksDB threads will automatically flush them to L0 + * SST files. The use case here is different: the MemTable may never get full and immutable given that the + * volume of data involved is relatively small. Further, we are constantly modifying the key-value pairs and + * generating WAL entries. The WAL file size can grow up to dozens of gigabytes without manual triggering of + * flush. + */ + if (ConfigStorage.this.estimateWalFileSize.get() >= messageStoreConfig.getRocksdbWalFileRollingThreshold()) { + ConfigStorage.this.flush(flushOptions); + estimateWalFileSize.set(0L); + } + + // Flush and Sync WAL if we have committed enough writes + if (writeOps >= messageStoreConfig.getRocksdbFlushWalFrequency() || onExit) { + stopwatch.reset().start(); + ConfigStorage.this.db.flushWal(true); + long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); + writeOpsCounter.getAndAdd(-writeOps); + lastSyncTime = System.currentTimeMillis(); + LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps); + return; + } + // Flush and Sync WAL if some writes are out there for a period of time + long elapsedTime = System.currentTimeMillis() - lastSyncTime; + if (elapsedTime > MAX_SYNC_INTERVAL_IN_MILLIS) { + stopwatch.reset().start(); + ConfigStorage.this.db.flushWal(true); + long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS); + LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps); + writeOpsCounter.getAndAdd(-writeOps); + lastSyncTime = System.currentTimeMillis(); + } + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java index 2c5d3677d88..1821c801cbc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java @@ -97,7 +97,7 @@ protected void removeConsumerOffset(String topicAtGroup) { // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey)); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { LOG.error("Failed to removeConsumerOffset, topicAtGroup={}", topicAtGroup, e); @@ -138,7 +138,7 @@ public void removeOffset(String group) { writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey)); MessageStore messageStore = brokerController.getMessageStore(); long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { LOG.error("Failed to consumer offsets by group={}", group, e); @@ -194,7 +194,7 @@ public void commitOffset(String clientHost, String group, String topic, int queu writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); MessageStore messageStore = brokerController.getMessageStore(); long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { LOG.error("Failed to commit consumer offset", e); @@ -394,7 +394,7 @@ public void commitPullOffset(String clientHost, String group, String topic, int try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.PULL_OFFSET, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { LOG.error("Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}", diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java index dea8a2d2c17..dd67871f184 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java @@ -137,8 +137,10 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion); configStorage.write(writeBatch); + // fdatasync on core metadata change + persist(); } catch (RocksDBException e) { log.error("update subscription group config error", e); } finally { @@ -163,7 +165,7 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.delete(ConfigHelper.readBytes(keyBuf)); long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion(); - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { log.error("Failed to remove subscription group config by group-name={}", groupName, e); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java index 4e36b087275..7991d704459 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java @@ -151,8 +151,10 @@ public void updateTopicConfig(final TopicConfig topicConfig) { try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer()); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion); configStorage.write(writeBatch); + // fdatasync on core metadata change + this.persist(); } catch (RocksDBException e) { log.error("Failed to update topic config", e); } finally { @@ -167,7 +169,7 @@ protected TopicConfig removeTopicConfig(String topicName) { try (WriteBatch writeBatch = new WriteBatch()) { writeBatch.delete(keyBuf.nioBuffer()); long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0; - ConfigHelper.stampDataVersion(writeBatch, dataVersion, stateMachineVersion); + ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion); configStorage.write(writeBatch); } catch (RocksDBException e) { log.error("Failed to delete topic config by topicName={}", topicName, e); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java index d7f46855e1a..132bd5c1a56 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2Test.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.broker.offset.ConsumerOffsetManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -44,6 +45,8 @@ public class ConsumerOffsetManagerV2Test { @Mock private BrokerController controller; + private MessageStoreConfig messageStoreConfig; + @Rule public TemporaryFolder tf = new TemporaryFolder(); @@ -60,7 +63,9 @@ public void setUp() throws IOException { Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig(); File configStoreDir = tf.newFolder(); - configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath()); + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); } @@ -84,7 +89,9 @@ public void testCommitOffset_Standard() { consumerOffsetManagerV2.getOffsetTable().clear(); Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); consumerOffsetManagerV2.load(); Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); } @@ -106,7 +113,9 @@ public void testCommitOffset_LMQ() { configStorage.shutdown(); + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); consumerOffsetManagerV2.load(); Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); } @@ -129,7 +138,9 @@ public void testCommitPullOffset_LMQ() { configStorage.shutdown(); + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); consumerOffsetManagerV2.load(); Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryPullOffset(group, topic, queueId)); } @@ -157,7 +168,10 @@ public void testRemoveByTopicAtGroup() { Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); configStorage.shutdown(); + + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); consumerOffsetManagerV2.load(); Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); Assert.assertEquals(queueOffset, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); @@ -184,7 +198,10 @@ public void testRemoveByGroup() { Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); configStorage.shutdown(); + + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + consumerOffsetManagerV2 = new ConsumerOffsetManagerV2(controller, configStorage); consumerOffsetManagerV2.load(); Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic, queueId)); Assert.assertEquals(-1L, consumerOffsetManagerV2.queryOffset(group, topic2, queueId)); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java index 6d436a7c4db..4ff8a81e60a 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2Test.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupRetryPolicyType; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -38,6 +39,9 @@ @RunWith(MockitoJUnitRunner.class) public class SubscriptionGroupManagerV2Test { + + private MessageStoreConfig messageStoreConfig; + private ConfigStorage configStorage; private SubscriptionGroupManagerV2 subscriptionGroupManagerV2; @@ -68,7 +72,9 @@ public void setUp() throws IOException { Mockito.doReturn(1L).when(messageStore).getStateMachineVersion(); File configStoreDir = tf.newFolder(); - configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath()); + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage); } @@ -98,7 +104,10 @@ public void testUpdateSubscriptionGroupConfig() { subscriptionGroupManagerV2.getSubscriptionGroupTable().clear(); configStorage.shutdown(); + + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage); subscriptionGroupManagerV2.load(); found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); Assert.assertEquals(subscriptionGroupConfig, found); @@ -132,7 +141,11 @@ public void testDeleteSubscriptionGroupConfig() { Assert.assertNull(found); configStorage.shutdown(); + + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); + + subscriptionGroupManagerV2 = new SubscriptionGroupManagerV2(controller, configStorage); subscriptionGroupManagerV2.load(); found = subscriptionGroupManagerV2.findSubscriptionGroupConfig(subscriptionGroupConfig.getGroupName()); Assert.assertNull(found); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java index 92c936b110a..731a1f538fb 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2Test.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.After; import org.junit.Assert; @@ -35,17 +36,19 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; - @RunWith(value = MockitoJUnitRunner.class) public class TopicConfigManagerV2Test { - private ConfigStorage configStorage; + private MessageStoreConfig messageStoreConfig; - private TopicConfigManagerV2 topicConfigManagerV2; + private ConfigStorage configStorage; @Mock private BrokerController controller; + @Mock + private MessageStore messageStore; + @Rule public TemporaryFolder tf = new TemporaryFolder(); @@ -61,17 +64,22 @@ public void setUp() throws IOException { BrokerConfig brokerConfig = new BrokerConfig(); Mockito.doReturn(brokerConfig).when(controller).getBrokerConfig(); - MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig = new MessageStoreConfig(); Mockito.doReturn(messageStoreConfig).when(controller).getMessageStoreConfig(); + Mockito.doReturn(messageStore).when(controller).getMessageStore(); File configStoreDir = tf.newFolder(); - configStorage = new ConfigStorage(configStoreDir.getAbsolutePath()); + messageStoreConfig.setStorePathRootDir(configStoreDir.getAbsolutePath()); + + configStorage = new ConfigStorage(messageStoreConfig); configStorage.start(); - topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); } @Test public void testUpdateTopicConfig() { + TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); + topicConfigManagerV2.load(); + TopicConfig topicConfig = new TopicConfig(); String topicName = "T1"; topicConfig.setTopicName(topicName); @@ -86,7 +94,9 @@ public void testUpdateTopicConfig() { topicConfigManagerV2.getTopicConfigTable().clear(); + configStorage = new ConfigStorage(messageStoreConfig); Assert.assertTrue(configStorage.start()); + topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); Assert.assertTrue(topicConfigManagerV2.load()); TopicConfig loaded = topicConfigManagerV2.selectTopicConfig(topicName); @@ -111,12 +121,15 @@ public void testRemoveTopicConfig() { topicConfig.setWriteQueueNums(4); topicConfig.setOrder(true); topicConfig.setTopicSysFlag(4); + TopicConfigManagerV2 topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); topicConfigManagerV2.updateTopicConfig(topicConfig); topicConfigManagerV2.removeTopicConfig(topicName); Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName)); Assert.assertTrue(configStorage.shutdown()); + configStorage = new ConfigStorage(messageStoreConfig); Assert.assertTrue(configStorage.start()); + topicConfigManagerV2 = new TopicConfigManagerV2(controller, configStorage); Assert.assertTrue(topicConfigManagerV2.load()); Assert.assertFalse(topicConfigManagerV2.containsTopic(topicName)); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java index 4b320eb53f3..6a805b04340 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/RocksdbTransferOffsetAndCqTest.java @@ -23,11 +23,11 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import org.apache.commons.collections.MapUtils; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager; import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.store.DefaultMessageStore; @@ -38,6 +38,7 @@ import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -135,6 +136,7 @@ public void testRocksdbCqWrite() throws RocksDBException { } RocksDBMessageStore kvStore = defaultMessageStore.getRocksDBMessageStore(); ConsumeQueueStoreInterface store = kvStore.getConsumeQueueStore(); + store.start(); ConsumeQueueInterface rocksdbCq = defaultMessageStore.getRocksDBMessageStore().findConsumeQueue(topic, queueId); ConsumeQueueInterface fileCq = defaultMessageStore.findConsumeQueue(topic, queueId); for (int i = 0; i < 200; i++) { @@ -142,13 +144,21 @@ public void testRocksdbCqWrite() throws RocksDBException { fileCq.putMessagePositionInfoWrapper(request); store.putMessagePositionInfoWrapper(request); } + Awaitility.await() + .pollInterval(100, TimeUnit.MILLISECONDS) + .atMost(3, TimeUnit.SECONDS) + .until(() -> rocksdbCq.getMaxOffsetInQueue() == 200); Pair unit = rocksdbCq.getCqUnitAndStoreTime(100); Pair unit1 = fileCq.getCqUnitAndStoreTime(100); - Assert.assertTrue(unit.getObject1().getPos() == unit1.getObject1().getPos()); + Assert.assertEquals(unit.getObject1().getPos(), unit1.getObject1().getPos()); } + /** + * No need to skip macOS platform. + * @return true if some platform is NOT a good fit for this test case. + */ private boolean notToBeExecuted() { - return MixAll.isMac(); + return false; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java index 28ed4e924c5..48ba4b8086c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java @@ -121,14 +121,16 @@ protected void initWriteOptions() { this.writeOptions = new WriteOptions(); this.writeOptions.setSync(false); this.writeOptions.setDisableWAL(true); - this.writeOptions.setNoSlowdown(true); + // https://github.com/facebook/rocksdb/wiki/Write-Stalls + this.writeOptions.setNoSlowdown(false); } protected void initAbleWalWriteOptions() { this.ableWalWriteOptions = new WriteOptions(); this.ableWalWriteOptions.setSync(false); this.ableWalWriteOptions.setDisableWAL(false); - this.ableWalWriteOptions.setNoSlowdown(true); + // https://github.com/facebook/rocksdb/wiki/Write-Stalls + this.ableWalWriteOptions.setNoSlowdown(false); } protected void initReadOptions() { @@ -363,7 +365,7 @@ public synchronized boolean start() { } if (postLoad()) { this.loaded = true; - LOGGER.info("start OK. {}", this.dbPath); + LOGGER.info("RocksDB[{}] starts OK", this.dbPath); this.closed = false; return true; } else { @@ -560,7 +562,15 @@ private String getStatusError(RocksDBException e) { public void statRocksdb(Logger logger) { try { + // Log Memory Usage + String blockCacheMemUsage = this.db.getProperty("rocksdb.block-cache-usage"); + String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem"); + String memTableMemUsage = this.db.getProperty("rocksdb.cur-size-all-mem-tables"); + String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage"); + logger.info("RocksDB Memory Usage: BlockCache: {}, IndexesAndFilterBlock: {}, MemTable: {}, BlocksPinnedByIterator: {}", + blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); + // Log file metadata by level List liveFileMetaDataList = this.getCompactionStatus(); if (liveFileMetaDataList == null || liveFileMetaDataList.isEmpty()) { return; @@ -570,21 +580,13 @@ public void statRocksdb(Logger logger) { StringBuilder sb = map.computeIfAbsent(metaData.level(), k -> new StringBuilder(256)); sb.append(new String(metaData.columnFamilyName(), StandardCharsets.UTF_8)).append(SPACE). append(metaData.fileName()).append(SPACE). - append("s: ").append(metaData.size()).append(SPACE). - append("a: ").append(metaData.numEntries()).append(SPACE). - append("r: ").append(metaData.numReadsSampled()).append(SPACE). - append("d: ").append(metaData.numDeletions()).append(SPACE). - append(metaData.beingCompacted()).append("\n"); + append("file-size: ").append(metaData.size()).append(SPACE). + append("number-of-entries: ").append(metaData.numEntries()).append(SPACE). + append("file-read-times: ").append(metaData.numReadsSampled()).append(SPACE). + append("deletions: ").append(metaData.numDeletions()).append(SPACE). + append("being-compacted: ").append(metaData.beingCompacted()).append("\n"); } - map.forEach((key, value) -> logger.info("level: {}\n{}", key, value.toString())); - - String blockCacheMemUsage = this.db.getProperty("rocksdb.block-cache-usage"); - String indexesAndFilterBlockMemUsage = this.db.getProperty("rocksdb.estimate-table-readers-mem"); - String memTableMemUsage = this.db.getProperty("rocksdb.cur-size-all-mem-tables"); - String blocksPinnedByIteratorMemUsage = this.db.getProperty("rocksdb.block-cache-pinned-usage"); - logger.info("MemUsage. blockCache: {}, indexesAndFilterBlock: {}, MemTable: {}, blocksPinnedByIterator: {}", - blockCacheMemUsage, indexesAndFilterBlockMemUsage, memTableMemUsage, blocksPinnedByIteratorMemUsage); } catch (Exception ignored) { } } diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java index a4ba35bd5ae..e3f6f22002e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java @@ -38,7 +38,7 @@ import org.rocksdb.util.SizeUnit; public class ConfigHelper { - public static ColumnFamilyOptions createConfigOptions() { + public static ColumnFamilyOptions createConfigColumnFamilyOptions() { BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). setFormatVersion(5). setIndexType(IndexType.kBinarySearch). @@ -46,7 +46,7 @@ public static ColumnFamilyOptions createConfigOptions() { setBlockSize(32 * SizeUnit.KB). setFilterPolicy(new BloomFilter(16, false)). // Indicating if we'd put index/filter blocks to the block cache. - setCacheIndexAndFilterBlocks(false). + setCacheIndexAndFilterBlocks(true). setCacheIndexAndFilterBlocksWithHighPriority(true). setPinL0FilterAndIndexBlocksInCache(false). setPinTopLevelIndexAndFilter(true). @@ -54,9 +54,8 @@ public static ColumnFamilyOptions createConfigOptions() { setWholeKeyFiltering(true); ColumnFamilyOptions options = new ColumnFamilyOptions(); - return options.setMaxWriteBufferNumber(2). - // MemTable size, MemTable(cache) -> immutable MemTable(cache) -> SST(disk) - setWriteBufferSize(8 * SizeUnit.MB). + return options.setMaxWriteBufferNumber(4). + setWriteBufferSize(64 * SizeUnit.MB). setMinWriteBufferNumberToMerge(1). setTableFormatConfig(blockBasedTableConfig). setMemTableConfig(new SkipListMemTableConfig()). @@ -67,17 +66,17 @@ public static ColumnFamilyOptions createConfigOptions() { setLevel0SlowdownWritesTrigger(8). setLevel0StopWritesTrigger(12). // The target file size for compaction. - setTargetFileSizeBase(64 * SizeUnit.MB). + setTargetFileSizeBase(64 * SizeUnit.MB). setTargetFileSizeMultiplier(2). // The upper-bound of the total size of L1 files in bytes - setMaxBytesForLevelBase(256 * SizeUnit.MB). + setMaxBytesForLevelBase(256 * SizeUnit.MB). setMaxBytesForLevelMultiplier(2). setMergeOperator(new StringAppendOperator()). setInplaceUpdateSupport(true); } public static DBOptions createConfigDBOptions() { - //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide + // Tune based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java DBOptions options = new DBOptions(); Statistics statistics = new Statistics(); @@ -86,10 +85,20 @@ public static DBOptions createConfigDBOptions() { setDbLogDir(getDBLogDir()). setInfoLogLevel(InfoLogLevel.INFO_LEVEL). setWalRecoveryMode(WALRecoveryMode.SkipAnyCorruptedRecords). + /* + * We use manual flush to achieve desired balance between reliability and performance: + * for metadata that matters, including {topic, subscription}-config changes, each write incurs a + * flush-and-sync to ensure reliability; for {commit, pull}-offset advancements, group-flush are offered for + * every N(configurable, 1024 by default) writes or aging of writes, similar to OS page-cache flush + * mechanism. + */ setManualWalFlush(true). - setMaxTotalWalSize(500 * SizeUnit.MB). - setWalSizeLimitMB(0). - setWalTtlSeconds(0). + // This option takes effect only when we have multiple column families + // https://github.com/facebook/rocksdb/issues/4180 + // setMaxTotalWalSize(1024 * SizeUnit.MB). + setDbWriteBufferSize(128 * SizeUnit.MB). + setBytesPerSync(SizeUnit.MB). + setWalBytesPerSync(SizeUnit.MB). setCreateIfMissing(true). setCreateMissingColumnFamilies(true). setMaxOpenFiles(-1). @@ -99,7 +108,6 @@ public static DBOptions createConfigDBOptions() { setAllowConcurrentMemtableWrite(false). setStatistics(statistics). setStatsDumpPeriodSec(600). - setAtomicFlush(true). setMaxBackgroundJobs(32). setMaxSubcompactions(4). setParanoidChecks(true). diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java index 3b924a6a0d2..5fd9bab2d77 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java @@ -70,7 +70,7 @@ protected boolean postLoad() { final List cfDescriptors = new ArrayList<>(); - ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigOptions(); + ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions(); this.cfOptions.add(defaultOptions); cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions)); cfDescriptors.add(new ColumnFamilyDescriptor(KV_DATA_VERSION_COLUMN_FAMILY_NAME, defaultOptions)); diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 7cf97465512..d30691908b2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -326,22 +326,26 @@ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBExcep boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); final List mappedFiles = this.mappedFileQueue.getMappedFiles(); if (!mappedFiles.isEmpty()) { - // Began to recover from the last third file - int index = mappedFiles.size() - 3; - if (index < 0) { - index = 0; + int index = mappedFiles.size() - 1; + while (index > 0) { + MappedFile mappedFile = mappedFiles.get(index); + if (mappedFile.getFileFromOffset() <= maxPhyOffsetOfConsumeQueue) { + // It's safe to recover from this mapped file + break; + } + index--; } + // TODO: Discuss if we need to load more commit-log mapped files into memory. MappedFile mappedFile = mappedFiles.get(index); ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); long processOffset = mappedFile.getFileFromOffset(); long mappedFileOffset = 0; long lastValidMsgPhyOffset = this.getConfirmOffset(); - // normal recover doesn't require dispatching - boolean doDispatch = false; while (true) { DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover, checkDupInfo); int size = dispatchRequest.getMsgSize(); + boolean doDispatch = dispatchRequest.getCommitLogOffset() > maxPhyOffsetOfConsumeQueue; // Normal data if (dispatchRequest.isSuccess() && size > 0) { lastValidMsgPhyOffset = processOffset + mappedFileOffset; diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index fe090e3fa2a..6dfdc0b1c84 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.store.StoreType; import org.apache.rocketmq.store.queue.BatchConsumeQueue; import org.rocksdb.CompressionType; +import org.rocksdb.util.SizeUnit; public class MessageStoreConfig { @@ -444,6 +445,13 @@ public class MessageStoreConfig { private String rocksdbCompressionType = CompressionType.LZ4_COMPRESSION.getLibraryName(); + /** + * Flush RocksDB WAL frequency, aka, flush WAL every N write ops. + */ + private int rocksdbFlushWalFrequency = 1024; + + private long rocksdbWalFileRollingThreshold = SizeUnit.GB; + public String getRocksdbCompressionType() { return rocksdbCompressionType; } @@ -1902,6 +1910,22 @@ public void setBottomMostCompressionTypeForConsumeQueueStore(String bottomMostCo this.bottomMostCompressionTypeForConsumeQueueStore = bottomMostCompressionTypeForConsumeQueueStore; } + public int getRocksdbFlushWalFrequency() { + return rocksdbFlushWalFrequency; + } + + public void setRocksdbFlushWalFrequency(int rocksdbFlushWalFrequency) { + this.rocksdbFlushWalFrequency = rocksdbFlushWalFrequency; + } + + public long getRocksdbWalFileRollingThreshold() { + return rocksdbWalFileRollingThreshold; + } + + public void setRocksdbWalFileRollingThreshold(long rocksdbWalFileRollingThreshold) { + this.rocksdbWalFileRollingThreshold = rocksdbWalFileRollingThreshold; + } + public int getSpinLockCollisionRetreatOptimalDegree() { return spinLockCollisionRetreatOptimalDegree; } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 0242ec23094..7e3aa70d026 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -30,12 +30,14 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; +import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageConst; @@ -84,6 +86,10 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private final OffsetInitializer offsetInitializer; + private final RocksGroupCommitService groupCommitService; + + private final AtomicReference serviceState = new AtomicReference<>(ServiceState.CREATE_JUST); + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); @@ -93,6 +99,7 @@ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); this.offsetInitializer = new OffsetInitializerRocksDBImpl(this); + this.groupCommitService = new RocksGroupCommitService(this); this.cqBBPairList = new ArrayList<>(16); this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY); for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) { @@ -123,14 +130,17 @@ private Pair getOffsetByteBufferPair() { @Override public void start() { - log.info("RocksDB ConsumeQueueStore start!"); - this.scheduledExecutorService.scheduleAtFixedRate(() -> { - this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); - }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); - - this.scheduledExecutorService.scheduleWithFixedDelay(() -> { - cleanDirty(messageStore.getTopicConfigs().keySet()); - }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); + if (serviceState.compareAndSet(ServiceState.CREATE_JUST, ServiceState.RUNNING)) { + log.info("RocksDB ConsumeQueueStore start!"); + this.groupCommitService.start(); + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); + }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleWithFixedDelay(() -> { + cleanDirty(messageStore.getTopicConfigs().keySet()); + }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); + } } private void cleanDirty(final Set existTopicSet) { @@ -165,18 +175,23 @@ public boolean loadAfterDestroy() { @Override public void recover() { - // ignored + start(); } @Override public boolean recoverConcurrently() { + start(); return true; } @Override public boolean shutdown() { - this.scheduledExecutorService.shutdown(); - return shutdownInner(); + if (serviceState.compareAndSet(ServiceState.RUNNING, ServiceState.SHUTDOWN_ALREADY)) { + this.groupCommitService.shutdown(); + this.scheduledExecutorService.shutdown(); + return shutdownInner(); + } + return true; } private boolean shutdownInner() { @@ -188,23 +203,25 @@ public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksD if (null == request) { return; } - // We are taking advantage of Atomic Flush, this operation is purely memory-based. - // batch and cache in Java heap does not make sense, instead, we should put the metadata into RocksDB immediately - // to optimized overall end-to-end latency. - putMessagePosition(request); + + try { + groupCommitService.putRequest(request); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } - public void putMessagePosition(DispatchRequest request) throws RocksDBException { + public void putMessagePosition(List requests) throws RocksDBException { final int maxRetries = 30; for (int i = 0; i < maxRetries; i++) { - if (putMessagePosition0(request)) { + if (putMessagePosition0(requests)) { if (this.isCQError) { this.messageStore.getRunningFlags().clearLogicsQueueError(); this.isCQError = false; } return; } else { - ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i); + ERROR_LOG.warn("Put cq Failed. retryTime: {}", i); try { Thread.sleep(100); } catch (InterruptedException ignored) { @@ -219,34 +236,43 @@ public void putMessagePosition(DispatchRequest request) throws RocksDBException throw new RocksDBException("put CQ Failed"); } - private boolean putMessagePosition0(DispatchRequest request) { + private boolean putMessagePosition0(List requests) { if (!this.rocksDBStorage.hold()) { return false; } try (WriteBatch writeBatch = new WriteBatch()) { + final int size = requests.size(); + if (size == 0) { + return true; + } long maxPhyOffset = 0; - DispatchEntry entry = DispatchEntry.from(request); - dispatch(entry, writeBatch); - dispatchLMQ(request, writeBatch); - - final int msgSize = request.getMsgSize(); - final long phyOffset = request.getCommitLogOffset(); - if (phyOffset + msgSize >= maxPhyOffset) { - maxPhyOffset = phyOffset + msgSize; + for (int i = size - 1; i >= 0; i--) { + final DispatchRequest request = requests.get(i); + DispatchEntry entry = DispatchEntry.from(request); + dispatch(entry, writeBatch); + dispatchLMQ(request, writeBatch); + + final int msgSize = request.getMsgSize(); + final long phyOffset = request.getCommitLogOffset(); + if (phyOffset + msgSize >= maxPhyOffset) { + maxPhyOffset = phyOffset + msgSize; + } } this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); this.rocksDBStorage.batchPut(writeBatch); + this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); - long storeTimeStamp = request.getStoreTimestamp(); + + long storeTimeStamp = requests.get(size - 1).getStoreTimestamp(); if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); } this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); - notifyMessageArrival(request); + notifyMessageArriveAndClear(requests); return true; } catch (Exception e) { ERROR_LOG.error("putMessagePosition0 failed.", e); @@ -311,9 +337,12 @@ private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteB } } - private void notifyMessageArrival(DispatchRequest request) { + private void notifyMessageArriveAndClear(List requests) { try { - this.messageStore.notifyMessageArriveIfNecessary(request); + for (DispatchRequest dp : requests) { + this.messageStore.notifyMessageArriveIfNecessary(dp); + } + requests.clear(); } catch (Exception e) { ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); } @@ -538,4 +567,8 @@ public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException } return super.getMaxOffset(topic, queueId); } + + public boolean isStopped() { + return ServiceState.SHUTDOWN_ALREADY == serviceState.get(); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java new file mode 100644 index 00000000000..e2f2c9ee2c1 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksGroupCommitService.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.store.DispatchRequest; +import org.rocksdb.RocksDBException; + +public class RocksGroupCommitService extends ServiceThread { + + private static final int MAX_BUFFER_SIZE = 100_000; + + private static final int PREFERRED_DISPATCH_REQUEST_COUNT = 256; + + private final LinkedBlockingQueue buffer; + + private final RocksDBConsumeQueueStore store; + + private final List requests = new ArrayList<>(PREFERRED_DISPATCH_REQUEST_COUNT); + + public RocksGroupCommitService(RocksDBConsumeQueueStore store) { + this.store = store; + this.buffer = new LinkedBlockingQueue<>(MAX_BUFFER_SIZE); + } + + @Override + public String getServiceName() { + return "RocksGroupCommit"; + } + + @Override + public void run() { + log.info("{} service started", this.getServiceName()); + while (!this.isStopped()) { + try { + this.waitForRunning(10); + this.doCommit(); + } catch (Exception e) { + log.warn("{} service has exception. ", this.getServiceName(), e); + } + } + log.info("{} service end", this.getServiceName()); + } + + public void putRequest(final DispatchRequest request) throws InterruptedException { + while (!buffer.offer(request, 3, TimeUnit.SECONDS)) { + log.warn("RocksGroupCommitService#buffer is full, 3s elapsed before space becomes available"); + } + this.wakeup(); + } + + private void doCommit() { + while (!buffer.isEmpty()) { + while (true) { + DispatchRequest dispatchRequest = buffer.poll(); + if (null != dispatchRequest) { + requests.add(dispatchRequest); + } + + if (requests.isEmpty()) { + // buffer has been drained + break; + } + + if (null == dispatchRequest || requests.size() >= PREFERRED_DISPATCH_REQUEST_COUNT) { + groupCommit(); + } + } + } + } + + private void groupCommit() { + while (!store.isStopped()) { + try { + // putMessagePosition will clear requests after consume queue building completion + store.putMessagePosition(requests); + break; + } catch (RocksDBException e) { + log.error("Failed to build consume queue in RocksDB", e); + } + } + } + +} diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java index 66f5cbd095d..2fac3bf485d 100644 --- a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java @@ -22,6 +22,7 @@ import org.rocksdb.BloomFilter; import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.CompactionOptionsUniversal; +import org.rocksdb.CompactionPriority; import org.rocksdb.CompactionStopStyle; import org.rocksdb.CompactionStyle; import org.rocksdb.CompressionType; @@ -79,6 +80,7 @@ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageSt setCompressionType(compressionType). setBottommostCompressionType(bottomMostCompressionType). setNumLevels(7). + setCompactionPriority(CompactionPriority.MinOverlappingRatio). setCompactionStyle(CompactionStyle.UNIVERSAL). setCompactionOptionsUniversal(compactionOption). setMaxCompactionBytes(100 * SizeUnit.GB). @@ -144,10 +146,8 @@ public static DBOptions createDBOptions() { setInfoLogLevel(InfoLogLevel.INFO_LEVEL). setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). setManualWalFlush(true). - setMaxTotalWalSize(0). - setWalSizeLimitMB(0). - setWalTtlSeconds(0). setCreateIfMissing(true). + setBytesPerSync(SizeUnit.MB). setCreateMissingColumnFamilies(true). setMaxOpenFiles(-1). setMaxLogFileSize(SizeUnit.GB). @@ -156,6 +156,7 @@ public static DBOptions createDBOptions() { setAllowConcurrentMemtableWrite(false). setStatistics(statistics). setAtomicFlush(true). + setCompactionReadaheadSize(4 * SizeUnit.MB). setMaxBackgroundJobs(32). setMaxSubcompactions(8). setParanoidChecks(true). From 78501c746123ce21622e18a3b6490220ace535d5 Mon Sep 17 00:00:00 2001 From: weihubeats Date: Mon, 2 Dec 2024 15:13:05 +0800 Subject: [PATCH 250/265] [ISSUE #9007] Fix client connection local ip is null in RemotingClient (#9008) --- .../apache/rocketmq/remoting/netty/NettyRemotingClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index b3042c9f8d3..6ac54aed6d2 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -74,6 +74,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.FutureUtils; +import org.apache.rocketmq.common.utils.NetworkUtil; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -1130,7 +1131,7 @@ class NettyConnectManageHandler extends ChannelDuplexHandler { @Override public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) throws Exception { - final String local = localAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(localAddress); + final String local = localAddress == null ? NetworkUtil.getLocalAddress() : RemotingHelper.parseSocketAddressAddr(localAddress); final String remote = remoteAddress == null ? "UNKNOWN" : RemotingHelper.parseSocketAddressAddr(remoteAddress); LOGGER.info("NETTY CLIENT PIPELINE: CONNECT {} => {}", local, remote); From e104c02d50758dacff00429d5e00458523c866f3 Mon Sep 17 00:00:00 2001 From: yx9o Date: Tue, 3 Dec 2024 09:56:40 +0800 Subject: [PATCH 251/265] [ISSUE #9014] Fix clusterAclConfigVersion command execution failed (#9017) --- .../rocketmq/broker/processor/AdminBrokerProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index fc3b6182731..4c341dde920 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -886,6 +886,12 @@ private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, Rem final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerAclConfigResponseHeader.class); + if (!brokerController.getBrokerConfig().isAclEnable()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("The broker does not enable acl."); + return response; + } + final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader(); try { From f4c498433d19e83510d4181ea9a63fbc7e3115eb Mon Sep 17 00:00:00 2001 From: rongtong Date: Wed, 4 Dec 2024 15:46:56 +0800 Subject: [PATCH 252/265] [ISSUE #7480] Fix the offset in the timerCheckPoint will not be corrected when the commitlog and consumeQueue are truncated (#7488) --- .../rocketmq/store/timer/TimerMessageStore.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 071b1c02192..fb166678e6a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -293,6 +293,19 @@ public void recover() { } currQueueOffset = Math.min(currQueueOffset, timerCheckpoint.getMasterTimerQueueOffset()); + ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, 0); + + // Correction based consume queue + if (cq != null && currQueueOffset < cq.getMinOffsetInQueue()) { + LOGGER.warn("Timer currQueueOffset:{} is smaller than minOffsetInQueue:{}", + currQueueOffset, cq.getMinOffsetInQueue()); + currQueueOffset = cq.getMinOffsetInQueue(); + } else if (cq != null && currQueueOffset > cq.getMaxOffsetInQueue()) { + LOGGER.warn("Timer currQueueOffset:{} is larger than maxOffsetInQueue:{}", + currQueueOffset, cq.getMaxOffsetInQueue()); + currQueueOffset = cq.getMaxOffsetInQueue(); + } + //check timer wheel currReadTimeMs = timerCheckpoint.getLastReadTimeMs(); long nextReadTimeMs = formatTimeMs( @@ -614,7 +627,7 @@ public void addMetric(MessageExt msg, int value) { return; } if (msg.getProperty(TIMER_ENQUEUE_MS) != null - && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { + && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { return; } // pass msg into addAndGet, for further more judgement extension. From 01a51231b3e8a0e72b2803d537bfbeb4f8e45719 Mon Sep 17 00:00:00 2001 From: Zhanhui Li Date: Thu, 5 Dec 2024 15:06:35 +0800 Subject: [PATCH 253/265] [ISSUE #9015] Sync SysFlag and message body inflation status; allow omit of message body (#9016) --- .../client/impl/consumer/ProcessQueue.java | 12 +++-- .../client/producer/ProduceAccumulator.java | 50 ++++++++++++------- .../hook/SendMessageOpenTracingHookImpl.java | 6 +-- .../trace/hook/SendMessageTraceHookImpl.java | 3 +- .../common/message/MessageDecoder.java | 4 +- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index 33e698b00c1..bc1b5eff2f9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -137,7 +137,7 @@ public boolean putMessage(final List msgs) { if (null == old) { validMsgCnt++; this.queueOffsetMax = msg.getQueueOffset(); - msgSize.addAndGet(msg.getBody().length); + msgSize.addAndGet(null == msg.getBody() ? 0 : msg.getBody().length); } } msgCount.addAndGet(validMsgCnt); @@ -198,7 +198,10 @@ public long removeMessage(final List msgs) { MessageExt prev = msgTreeMap.remove(msg.getQueueOffset()); if (prev != null) { removedCnt--; - msgSize.addAndGet(-msg.getBody().length); + long bodySize = null == msg.getBody() ? 0 : msg.getBody().length; + if (bodySize > 0) { + msgSize.addAndGet(-bodySize); + } } } if (msgCount.addAndGet(removedCnt) == 0) { @@ -270,7 +273,10 @@ public long commit() { msgSize.set(0); } else { for (MessageExt msg : this.consumingMsgOrderlyTreeMap.values()) { - msgSize.addAndGet(-msg.getBody().length); + int bodySize = null == msg.getBody() ? 0 : msg.getBody().length; + if (bodySize > 0) { + msgSize.addAndGet(-bodySize); + } } } this.consumingMsgOrderlyTreeMap.clear(); diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java b/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java index 46dfcf71d29..809830e4641 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/ProduceAccumulator.java @@ -52,9 +52,9 @@ public class ProduceAccumulator { private final Logger log = LoggerFactory.getLogger(DefaultMQProducer.class); private final GuardForSyncSendService guardThreadForSyncSend; private final GuardForAsyncSendService guardThreadForAsyncSend; - private Map syncSendBatchs = new ConcurrentHashMap(); - private Map asyncSendBatchs = new ConcurrentHashMap(); - private AtomicLong currentlyHoldSize = new AtomicLong(0); + private final Map syncSendBatchs = new ConcurrentHashMap(); + private final Map asyncSendBatchs = new ConcurrentHashMap(); + private final AtomicLong currentlyHoldSize = new AtomicLong(0); private final String instanceName; public ProduceAccumulator(String instanceName) { @@ -70,11 +70,13 @@ public GuardForSyncSendService(String clientInstanceName) { serviceName = String.format("Client_%s_GuardForSyncSend", clientInstanceName); } - @Override public String getServiceName() { + @Override + public String getServiceName() { return serviceName; } - @Override public void run() { + @Override + public void run() { log.info(this.getServiceName() + " service started"); while (!this.isStopped()) { @@ -115,11 +117,13 @@ public GuardForAsyncSendService(String clientInstanceName) { serviceName = String.format("Client_%s_GuardForAsyncSend", clientInstanceName); } - @Override public String getServiceName() { + @Override + public String getServiceName() { return serviceName; } - @Override public void run() { + @Override + public void run() { log.info(this.getServiceName() + " service started"); while (!this.isStopped()) { @@ -276,7 +280,10 @@ void send(Message msg, MessageQueue mq, boolean tryAddMessage(Message message) { synchronized (currentlyHoldSize) { if (currentlyHoldSize.get() < totalHoldSize) { - currentlyHoldSize.addAndGet(message.getBody().length); + int bodySize = null == message.getBody() ? 0 : message.getBody().length; + if (bodySize > 0) { + currentlyHoldSize.addAndGet(bodySize); + } return true; } else { return false; @@ -305,7 +312,8 @@ public AggregateKey(String topic, MessageQueue mq, boolean waitStoreMsgOK, Strin this.tag = tag; } - @Override public boolean equals(Object o) { + @Override + public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) @@ -314,7 +322,8 @@ public AggregateKey(String topic, MessageQueue mq, boolean waitStoreMsgOK, Strin return waitStoreMsgOK == key.waitStoreMsgOK && topic.equals(key.topic) && Objects.equals(mq, key.mq) && Objects.equals(tag, key.tag); } - @Override public int hashCode() { + @Override + public int hashCode() { return Objects.hash(topic, mq, waitStoreMsgOK, tag); } } @@ -324,7 +333,7 @@ private class MessageAccumulation { private LinkedList messages; private LinkedList sendCallbacks; private Set keys; - private AtomicBoolean closed; + private final AtomicBoolean closed; private SendResult[] sendResults; private AggregateKey aggregateKey; private AtomicInteger messagesSize; @@ -351,8 +360,7 @@ private boolean readyToSend() { return false; } - public int add( - Message msg) throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + public int add(Message msg) throws InterruptedException, MQBrokerException, RemotingException, MQClientException { int ret = -1; synchronized (this.closed) { if (this.closed.get()) { @@ -360,7 +368,10 @@ public int add( } ret = this.count++; this.messages.add(msg); - messagesSize.addAndGet(msg.getBody().length); + int bodySize = null == msg.getBody() ? 0 : msg.getBody().length; + if (bodySize > 0) { + messagesSize.addAndGet(bodySize); + } String msgKeys = msg.getKeys(); if (msgKeys != null) { this.keys.addAll(Arrays.asList(msgKeys.split(MessageConst.KEY_SEPARATOR))); @@ -388,7 +399,10 @@ public boolean add(Message msg, this.count++; this.messages.add(msg); this.sendCallbacks.add(sendCallback); - messagesSize.getAndAdd(msg.getBody().length); + int bodySize = null == msg.getBody() ? 0 : msg.getBody().length; + if (bodySize > 0) { + messagesSize.addAndGet(bodySize); + } } if (readyToSend()) { this.send(sendCallback); @@ -472,7 +486,8 @@ private void send(SendCallback sendCallback) { if (defaultMQProducer != null) { final int size = messagesSize.get(); defaultMQProducer.sendDirect(messageBatch, aggregateKey.mq, new SendCallback() { - @Override public void onSuccess(SendResult sendResult) { + @Override + public void onSuccess(SendResult sendResult) { try { splitSendResults(sendResult); int i = 0; @@ -490,7 +505,8 @@ private void send(SendCallback sendCallback) { } } - @Override public void onException(Throwable e) { + @Override + public void onException(Throwable e) { for (SendCallback v : sendCallbacks) { v.onException(e); } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java index 3cb64933849..0f828f2b4e1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageOpenTracingHookImpl.java @@ -48,8 +48,8 @@ public void sendMessageBefore(SendMessageContext context) { } Message msg = context.getMessage(); Tracer.SpanBuilder spanBuilder = tracer - .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic()) - .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER); + .buildSpan(TraceConstants.TO_PREFIX + msg.getTopic()) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_PRODUCER); SpanContext spanContext = tracer.extract(Format.Builtin.TEXT_MAP, new TextMapAdapter(msg.getProperties())); if (spanContext != null) { spanBuilder.asChildOf(spanContext); @@ -62,7 +62,7 @@ public void sendMessageBefore(SendMessageContext context) { span.setTag(TraceConstants.ROCKETMQ_KEYS, msg.getKeys()); span.setTag(TraceConstants.ROCKETMQ_STORE_HOST, context.getBrokerAddr()); span.setTag(TraceConstants.ROCKETMQ_MSG_TYPE, context.getMsgType().name()); - span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, msg.getBody().length); + span.setTag(TraceConstants.ROCKETMQ_BODY_LENGTH, null == msg.getBody() ? 0 : msg.getBody().length); context.setMqTraceContext(span); } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java index dba04b593f2..61738928bb3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java @@ -58,7 +58,8 @@ public void sendMessageBefore(SendMessageContext context) { traceBean.setTags(context.getMessage().getTags()); traceBean.setKeys(context.getMessage().getKeys()); traceBean.setStoreHost(context.getBrokerAddr()); - traceBean.setBodyLength(context.getMessage().getBody().length); + int bodyLength = null == context.getMessage().getBody() ? 0 : context.getMessage().getBody().length; + traceBean.setBodyLength(bodyLength); traceBean.setMsgType(context.getMsgType()); traceContext.getTraceBeans().add(traceBean); } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index f5491e192af..713f9405ea9 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -516,13 +516,15 @@ public static MessageExt decode( } } - // uncompress body + // inflate body if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { Compressor compressor = CompressorFactory.getCompressor(MessageSysFlag.getCompressionType(sysFlag)); body = compressor.decompress(body); + sysFlag &= ~MessageSysFlag.COMPRESSED_FLAG; } msgExt.setBody(body); + msgExt.setSysFlag(sysFlag); } else { byteBuffer.position(byteBuffer.position() + bodyLen); } From d1fd7af3f12e1bfa30c7baa7c3f687168a9f5dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Thu, 5 Dec 2024 17:49:03 +0800 Subject: [PATCH 254/265] [ISSUE #8979] Add configurable switch for timer message retry logic (#8980) --- .../store/config/MessageStoreConfig.java | 9 ++ .../store/timer/TimerMessageStore.java | 121 ++++++++++-------- .../store/timer/TimerMessageStoreTest.java | 87 ++++++++++--- 3 files changed, 149 insertions(+), 68 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 6dfdc0b1c84..0ea58415487 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -99,6 +99,7 @@ public class MessageStoreConfig { private boolean timerSkipUnknownError = false; private boolean timerWarmEnable = false; private boolean timerStopDequeue = false; + private boolean timerEnableRetryUntilSuccess = false; private int timerCongestNumEachSlot = Integer.MAX_VALUE; private int timerMetricSmallThreshold = 1000000; @@ -1689,6 +1690,14 @@ public void setTimerSkipUnknownError(boolean timerSkipUnknownError) { this.timerSkipUnknownError = timerSkipUnknownError; } + public boolean isTimerEnableRetryUntilSuccess() { + return timerEnableRetryUntilSuccess; + } + + public void setTimerEnableRetryUntilSuccess(boolean timerEnableRetryUntilSuccess) { + this.timerEnableRetryUntilSuccess = timerEnableRetryUntilSuccess; + } + public boolean isTimerWarmEnable() { return timerWarmEnable; } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index fb166678e6a..2b14618eede 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1097,46 +1097,44 @@ public int doPut(MessageExtBrokerInner message, boolean roll) throws Exception { putMessageResult = messageStore.putMessage(message); } - int retryNum = 0; - while (retryNum < 3) { - if (null == putMessageResult || null == putMessageResult.getPutMessageStatus()) { - retryNum++; - } else { - switch (putMessageResult.getPutMessageStatus()) { - case PUT_OK: - if (brokerStatsManager != null) { - this.brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1); - if (putMessageResult.getAppendMessageResult() != null) { - this.brokerStatsManager.incTopicPutSize(message.getTopic(), - putMessageResult.getAppendMessageResult().getWroteBytes()); - } - this.brokerStatsManager.incBrokerPutNums(message.getTopic(), 1); + if (putMessageResult != null && putMessageResult.getPutMessageStatus() != null) { + switch (putMessageResult.getPutMessageStatus()) { + case PUT_OK: + if (brokerStatsManager != null) { + brokerStatsManager.incTopicPutNums(message.getTopic(), 1, 1); + if (putMessageResult.getAppendMessageResult() != null) { + brokerStatsManager.incTopicPutSize(message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes()); } - return PUT_OK; - case SERVICE_NOT_AVAILABLE: - return PUT_NEED_RETRY; - case MESSAGE_ILLEGAL: - case PROPERTIES_SIZE_EXCEEDED: + brokerStatsManager.incBrokerPutNums(message.getTopic(), 1); + } + return PUT_OK; + + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + case WHEEL_TIMER_NOT_ENABLE: + case WHEEL_TIMER_MSG_ILLEGAL: + return PUT_NO_RETRY; + + case SERVICE_NOT_AVAILABLE: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case OS_PAGE_CACHE_BUSY: + case CREATE_MAPPED_FILE_FAILED: + case SLAVE_NOT_AVAILABLE: + return PUT_NEED_RETRY; + + case UNKNOWN_ERROR: + default: + if (storeConfig.isTimerSkipUnknownError()) { + LOGGER.warn("Skipping message due to unknown error, msg: {}", message); return PUT_NO_RETRY; - case CREATE_MAPPED_FILE_FAILED: - case FLUSH_DISK_TIMEOUT: - case FLUSH_SLAVE_TIMEOUT: - case OS_PAGE_CACHE_BUSY: - case SLAVE_NOT_AVAILABLE: - case UNKNOWN_ERROR: - default: - retryNum++; - } - } - Thread.sleep(50); - if (escapeBridgeHook != null) { - putMessageResult = escapeBridgeHook.apply(message); - } else { - putMessageResult = messageStore.putMessage(message); + } else { + holdMomentForUnknownError(); + return PUT_NEED_RETRY; + } } - LOGGER.warn("Retrying to do put timer msg retryNum:{} putRes:{} msg:{}", retryNum, putMessageResult, message); } - return PUT_NO_RETRY; + return PUT_NEED_RETRY; } public MessageExtBrokerInner convertMessage(MessageExt msgExt, boolean needRoll) { @@ -1471,7 +1469,6 @@ protected boolean isState(int state) { } public class TimerDequeuePutMessageService extends AbstractStateService { - @Override public String getServiceName() { return getServiceThreadName() + this.getClass().getSimpleName(); @@ -1481,6 +1478,7 @@ public String getServiceName() { public void run() { setState(AbstractStateService.START); TimerMessageStore.LOGGER.info(this.getServiceName() + " service start"); + while (!this.isStopped() || dequeuePutQueue.size() != 0) { try { setState(AbstractStateService.WAITING); @@ -1488,41 +1486,63 @@ public void run() { if (null == tr) { continue; } + setState(AbstractStateService.RUNNING); - boolean doRes = false; boolean tmpDequeueChangeFlag = false; + try { - while (!isStopped() && !doRes) { + while (!isStopped()) { if (!isRunningDequeue()) { dequeueStatusChangeFlag = true; tmpDequeueChangeFlag = true; break; } + try { perfCounterTicks.startTick(DEQUEUE_PUT); + MessageExt msgExt = tr.getMsg(); DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt)); + if (tr.getEnqueueTime() == Long.MAX_VALUE) { - // never enqueue, mark it. + // Never enqueue, mark it. MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE)); } + addMetric(msgExt, -1); MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic())); - doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); - while (!doRes && !isStopped()) { - if (!isRunningDequeue()) { - dequeueStatusChangeFlag = true; - tmpDequeueChangeFlag = true; - break; + + boolean processed = false; + int retryCount = 0; + + while (!processed && !isStopped()) { + int result = doPut(msg, needRoll(tr.getMagic())); + + if (result == PUT_OK) { + processed = true; + } else if (result == PUT_NO_RETRY) { + TimerMessageStore.LOGGER.warn("Skipping message due to unrecoverable error. Msg: {}", msg); + processed = true; + } else { + retryCount++; + // Without enabling TimerEnableRetryUntilSuccess, messages will retry up to 3 times before being discarded + if (!storeConfig.isTimerEnableRetryUntilSuccess() && retryCount >= 3) { + TimerMessageStore.LOGGER.error("Message processing failed after {} retries. Msg: {}", retryCount, msg); + processed = true; + } else { + Thread.sleep(500L * precisionMs / 1000); + TimerMessageStore.LOGGER.warn("Retrying to process message. Retry count: {}, Msg: {}", retryCount, msg); + } } - doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); - Thread.sleep(500L * precisionMs / 1000); } + perfCounterTicks.endTick(DEQUEUE_PUT); + break; + } catch (Throwable t) { - LOGGER.info("Unknown error", t); + TimerMessageStore.LOGGER.info("Unknown error", t); if (storeConfig.isTimerSkipUnknownError()) { - doRes = true; + break; } else { holdMomentForUnknownError(); } @@ -1531,7 +1551,6 @@ public void run() { } finally { tr.idempotentRelease(!tmpDequeueChangeFlag); } - } catch (Throwable e) { TimerMessageStore.LOGGER.error("Error occurred in " + getServiceName(), e); } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java index 4ce3985f6c9..52e58efde23 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -30,6 +31,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.common.BrokerConfig; @@ -40,23 +42,26 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.store.ConsumeQueue; -import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.GetMessageResult; -import org.apache.rocketmq.store.GetMessageStatus; -import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.apache.rocketmq.store.ConsumeQueue; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageArrivingListener; import org.junit.After; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -65,10 +70,16 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.times; public class TimerMessageStoreTest { private final byte[] msgBody = new byte[1024]; private static MessageStore messageStore; + private MessageStore mockMessageStore; private SocketAddress bornHost; private SocketAddress storeHost; @@ -100,21 +111,23 @@ public void init() throws Exception { storeConfig.setTimerInterceptDelayLevel(true); storeConfig.setTimerPrecisionMs(precisionMs); + mockMessageStore = Mockito.mock(MessageStore.class); messageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("TimerTest",false), new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); boolean load = messageStore.load(); assertTrue(load); messageStore.start(); } - public TimerMessageStore createTimerMessageStore(String rootDir) throws IOException { + public TimerMessageStore createTimerMessageStore(String rootDir , boolean needMock) throws IOException { if (null == rootDir) { rootDir = StoreTestUtils.createBaseDir(); } TimerCheckpoint timerCheckpoint = new TimerCheckpoint(rootDir + File.separator + "config" + File.separator + "timercheck"); TimerMetrics timerMetrics = new TimerMetrics(rootDir + File.separator + "config" + File.separator + "timermetrics"); - TimerMessageStore timerMessageStore = new TimerMessageStore(messageStore, storeConfig, timerCheckpoint, timerMetrics, null); - messageStore.setTimerMessageStore(timerMessageStore); + MessageStore ms = needMock ? mockMessageStore : messageStore; + TimerMessageStore timerMessageStore = new TimerMessageStore(ms, storeConfig, timerCheckpoint, timerMetrics, null); + ms.setTimerMessageStore(timerMessageStore); baseDirs.add(rootDir); timerStores.add(timerMessageStore); @@ -170,7 +183,7 @@ public void testPutTimerMessage() throws Exception { Assume.assumeFalse(MixAll.isWindows()); String topic = "TimerTest_testPutTimerMessage"; - final TimerMessageStore timerMessageStore = createTimerMessageStore(null); + final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false); timerMessageStore.load(); timerMessageStore.start(true); @@ -212,12 +225,52 @@ public Boolean call() { } } + @Test + public void testRetryUntilSuccess() throws Exception { + storeConfig.setTimerEnableRetryUntilSuccess(true); + TimerMessageStore timerMessageStore = createTimerMessageStore(null , true); + timerMessageStore.load(); + timerMessageStore.setShouldRunningDequeue(true); + Field stateField = TimerMessageStore.class.getDeclaredField("state"); + stateField.setAccessible(true); + stateField.set(timerMessageStore, TimerMessageStore.RUNNING); + + MessageExtBrokerInner msg = buildMessage(3000L, "TestRetry", true); + transformTimerMessage(timerMessageStore, msg); + TimerRequest timerRequest = new TimerRequest(100, 200, 3000, System.currentTimeMillis(), 0, msg); + boolean offered = timerMessageStore.dequeuePutQueue.offer(timerRequest); + assertTrue(offered); + assertFalse(timerMessageStore.dequeuePutQueue.isEmpty()); + + // If enableRetryUntilSuccess is set and putMessage return NEED_RETRY type, the message should be retried until success. + when(mockMessageStore.putMessage(any(MessageExtBrokerInner.class))) + .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, null)) + .thenReturn(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, null)) + .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null)) + .thenReturn(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null)) + .thenReturn(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null)) + .thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + + final CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try { + timerMessageStore.getDequeuePutMessageServices()[0].run(); + } finally { + latch.countDown(); + } + }).start(); + latch.await(10, TimeUnit.SECONDS); + + assertTrue(timerMessageStore.dequeuePutQueue.isEmpty()); + verify(mockMessageStore, times(6)).putMessage(any(MessageExtBrokerInner.class)); + } + @Test public void testTimerFlowControl() throws Exception { String topic = "TimerTest_testTimerFlowControl"; storeConfig.setTimerCongestNumEachSlot(100); - TimerMessageStore timerMessageStore = createTimerMessageStore(null); + TimerMessageStore timerMessageStore = createTimerMessageStore(null , false); timerMessageStore.load(); timerMessageStore.start(true); @@ -264,7 +317,7 @@ public void testPutExpiredTimerMessage() throws Exception { String topic = "TimerTest_testPutExpiredTimerMessage"; - TimerMessageStore timerMessageStore = createTimerMessageStore(null); + TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false); timerMessageStore.load(); timerMessageStore.start(true); @@ -288,7 +341,7 @@ public void testPutExpiredTimerMessage() throws Exception { public void testDeleteTimerMessage() throws Exception { String topic = "TimerTest_testDeleteTimerMessage"; - TimerMessageStore timerMessageStore = createTimerMessageStore(null); + TimerMessageStore timerMessageStore = createTimerMessageStore(null ,false); timerMessageStore.load(); timerMessageStore.start(true); @@ -325,7 +378,7 @@ public void testDeleteTimerMessage() throws Exception { public void testPutDeleteTimerMessage() throws Exception { String topic = "TimerTest_testPutDeleteTimerMessage"; - final TimerMessageStore timerMessageStore = createTimerMessageStore(null); + final TimerMessageStore timerMessageStore = createTimerMessageStore(null , false); timerMessageStore.load(); timerMessageStore.start(true); @@ -372,7 +425,7 @@ public void testStateAndRecover() throws Exception { final String topic = "TimerTest_testStateAndRecover"; String base = StoreTestUtils.createBaseDir(); - final TimerMessageStore first = createTimerMessageStore(base); + final TimerMessageStore first = createTimerMessageStore(base , false); first.load(); first.start(true); @@ -417,7 +470,7 @@ public Boolean call() { first.getTimerWheel().flush(); first.shutdown(); - final TimerMessageStore second = createTimerMessageStore(base); + final TimerMessageStore second = createTimerMessageStore(base , false); second.debug = true; assertTrue(second.load()); assertEquals(msgNum, second.getQueueOffset()); @@ -446,7 +499,7 @@ public Boolean call() { public void testMaxDelaySec() throws Exception { String topic = "TimerTest_testMaxDelaySec"; - TimerMessageStore first = createTimerMessageStore(null); + TimerMessageStore first = createTimerMessageStore(null , false); first.load(); first.start(true); @@ -468,7 +521,7 @@ public void testRollMessage() throws Exception { storeConfig.setTimerRollWindowSlot(2); String topic = "TimerTest_testRollMessage"; - TimerMessageStore timerMessageStore = createTimerMessageStore(null); + TimerMessageStore timerMessageStore = createTimerMessageStore(null , false); timerMessageStore.load(); timerMessageStore.start(true); From 4ea8d1ff6c3c306454ec23d4e4bba4db0b9987fc Mon Sep 17 00:00:00 2001 From: asapple <65564735+asapple@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:59:46 +0800 Subject: [PATCH 255/265] refactor(LmqBrokerStatsManager): extract common method to eliminate duplicate logic (#9034) Extracted the duplicated logic of replacing group and topic with LMQ_PREFIX based on configuration into a common method, improving code structure and maintainability. --- .../store/stats/LmqBrokerStatsManager.java | 116 +++--------------- 1 file changed, 17 insertions(+), 99 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java index 20ed8793318..4caea19f567 100644 --- a/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java +++ b/store/src/main/java/org/apache/rocketmq/store/stats/LmqBrokerStatsManager.java @@ -30,139 +30,57 @@ public LmqBrokerStatsManager(BrokerConfig brokerConfig) { @Override public void incGroupGetNums(final String group, final String topic, final int incValue) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incGroupGetNums(lmqGroup, lmqTopic, incValue); + super.incGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue); } @Override public void incGroupGetSize(final String group, final String topic, final int incValue) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incGroupGetSize(lmqGroup, lmqTopic, incValue); + super.incGroupGetSize(getAdjustedGroup(group), getAdjustedTopic(topic), incValue); } @Override public void incGroupAckNums(final String group, final String topic, final int incValue) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incGroupAckNums(lmqGroup, lmqTopic, incValue); + super.incGroupAckNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue); } @Override public void incGroupCkNums(final String group, final String topic, final int incValue) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incGroupCkNums(lmqGroup, lmqTopic, incValue); + super.incGroupCkNums(getAdjustedGroup(group), getAdjustedTopic(topic), incValue); } @Override public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incGroupGetLatency(lmqGroup, lmqTopic, queueId, incValue); + super.incGroupGetLatency(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, incValue); } @Override public void incSendBackNums(final String group, final String topic) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.incSendBackNums(lmqGroup, lmqTopic); + super.incSendBackNums(getAdjustedGroup(group), getAdjustedTopic(topic)); } @Override public double tpsGroupGetNums(final String group, final String topic) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - return super.tpsGroupGetNums(lmqGroup, lmqTopic); + return super.tpsGroupGetNums(getAdjustedGroup(group), getAdjustedTopic(topic)); } @Override public void recordDiskFallBehindTime(final String group, final String topic, final int queueId, final long fallBehind) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.recordDiskFallBehindTime(lmqGroup, lmqTopic, queueId, fallBehind); + super.recordDiskFallBehindTime(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind); } @Override public void recordDiskFallBehindSize(final String group, final String topic, final int queueId, final long fallBehind) { - String lmqGroup = group; - String lmqTopic = topic; - if (!brokerConfig.isEnableLmqStats()) { - if (MixAll.isLmq(group)) { - lmqGroup = MixAll.LMQ_PREFIX; - } - if (MixAll.isLmq(topic)) { - lmqTopic = MixAll.LMQ_PREFIX; - } - } - super.recordDiskFallBehindSize(lmqGroup, lmqTopic, queueId, fallBehind); + super.recordDiskFallBehindSize(getAdjustedGroup(group), getAdjustedTopic(topic), queueId, fallBehind); + } + + private String getAdjustedGroup(String group) { + return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(group) ? MixAll.LMQ_PREFIX : group; + } + + private String getAdjustedTopic(String topic) { + return !brokerConfig.isEnableLmqStats() && MixAll.isLmq(topic) ? MixAll.LMQ_PREFIX : topic; } } From 4571a853a2e0b5be4e4d94d9a9865f235a2c9c76 Mon Sep 17 00:00:00 2001 From: irotuk Date: Mon, 9 Dec 2024 14:32:37 +0800 Subject: [PATCH 256/265] reduce duplicate calls (#9037) Co-authored-by: noreply --- .../main/java/org/apache/rocketmq/srvutil/FileWatchService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java index 06c301bec9c..0c0690da5ce 100644 --- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/FileWatchService.java @@ -64,7 +64,7 @@ public void run0() { this.waitForRunning(WATCH_INTERVAL); for (Map.Entry entry : currentHash.entrySet()) { String newHash = md5Digest(entry.getKey()); - if (!newHash.equals(currentHash.get(entry.getKey()))) { + if (!newHash.equals(entry.getValue())) { entry.setValue(newHash); listener.onChanged(entry.getKey()); } From bfb3d17ef11291461685d137eab7f72abb9a1d8d Mon Sep 17 00:00:00 2001 From: imzs Date: Mon, 9 Dec 2024 16:56:12 +0800 Subject: [PATCH 257/265] [ISSUE #8974] Support recalling of delay message (#8975) --- .../acl/plain/PlainAccessResource.java | 7 + .../acl/plain/PlainAccessResourceTest.java | 37 +++ .../DefaultAuthorizationContextBuilder.java | 9 + ...efaultAuthorizationContextBuilderTest.java | 31 +++ .../rocketmq/broker/BrokerController.java | 9 + .../processor/RecallMessageProcessor.java | 184 +++++++++++++ .../processor/SendMessageProcessor.java | 17 ++ .../processor/RecallMessageProcessorTest.java | 241 ++++++++++++++++++ .../processor/SendMessageProcessorTest.java | 65 +++++ .../rocketmq/client/impl/MQClientAPIImpl.java | 48 ++++ .../client/impl/mqclient/MQClientAPIExt.java | 22 ++ .../impl/producer/DefaultMQProducerImpl.java | 37 +++ .../client/producer/DefaultMQProducer.java | 9 + .../rocketmq/client/producer/MQProducer.java | 3 + .../rocketmq/client/producer/SendResult.java | 11 +- .../client/trace/TraceDataEncoder.java | 24 ++ .../rocketmq/client/trace/TraceType.java | 1 + .../hook/DefaultRecallMessageTraceHook.java | 85 ++++++ .../client/impl/MQClientAPIImplTest.java | 76 ++++++ .../impl/mqclient/MQClientAPIExtTest.java | 47 ++++ .../selector/DefaultMQProducerImplTest.java | 43 +++- .../apache/rocketmq/common/BrokerConfig.java | 10 + .../common/producer/RecallMessageHandle.java | 96 +++++++ .../producer/RecallMessageHandleTest.java | 68 +++++ .../grpc/interceptor/RequestMapping.java | 2 + .../grpc/v2/DefaultGrpcMessingActivity.java | 11 + .../grpc/v2/GrpcMessagingApplication.java | 21 ++ .../proxy/grpc/v2/GrpcMessingActivity.java | 4 + .../v2/producer/RecallMessageActivity.java | 63 +++++ .../grpc/v2/producer/SendMessageActivity.java | 1 + .../processor/DefaultMessagingProcessor.java | 6 + .../proxy/processor/MessagingProcessor.java | 7 + .../proxy/processor/ProducerProcessor.java | 30 +++ .../remoting/RemotingProtocolServer.java | 4 + .../activity/RecallMessageActivity.java | 53 ++++ .../message/ClusterMessageService.java | 11 + .../service/message/LocalMessageService.java | 29 +++ .../proxy/service/message/MessageService.java | 8 + .../producer/RecallMessageActivityTest.java | 86 +++++++ .../processor/ProducerProcessorTest.java | 39 +++ .../activity/RecallMessageActivityTest.java | 109 ++++++++ .../message/LocalMessageServiceTest.java | 33 +++ .../remoting/protocol/RequestCode.java | 1 + .../header/RecallMessageRequestHeader.java | 78 ++++++ .../header/RecallMessageResponseHeader.java | 38 +++ .../header/SendMessageResponseHeader.java | 15 ++ .../store/timer/TimerMessageStore.java | 8 +- .../store/timer/TimerMessageStoreTest.java | 45 +++- test/BUILD.bazel | 2 + .../rocketmq/test/grpc/v2/ClusterGrpcIT.java | 5 + .../rocketmq/test/grpc/v2/GrpcBaseIT.java | 66 +++++ .../rocketmq/test/grpc/v2/LocalGrpcIT.java | 5 + .../test/recall/RecallWithTraceIT.java | 104 ++++++++ .../recall/SendAndRecallDelayMessageIT.java | 193 ++++++++++++++ 54 files changed, 2253 insertions(+), 4 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivity.java create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java create mode 100644 proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java create mode 100644 remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java create mode 100644 test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java create mode 100644 test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java index ef05fa6adbb..e45f99799d3 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java @@ -26,6 +26,7 @@ import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.RecallMessageRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.Resource; import apache.rocketmq.v2.SendMessageRequest; @@ -128,6 +129,9 @@ public static PlainAccessResource parse(RemotingCommand request, String remoteAd final String topicV2 = request.getExtFields().get("b"); accessResource.addResourceAndPerm(topicV2, PlainAccessResource.isRetryTopic(topicV2) ? Permission.SUB : Permission.PUB); break; + case RequestCode.RECALL_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.PUB); + break; case RequestCode.CONSUMER_SEND_MSG_BACK: accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); break; @@ -232,6 +236,9 @@ public static PlainAccessResource parse(GeneratedMessageV3 messageV3, Authentica } } accessResource.addResourceAndPerm(topic, Permission.PUB); + } else if (RecallMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) { + RecallMessageRequest request = (RecallMessageRequest) messageV3; + accessResource.addResourceAndPerm(request.getTopic(), Permission.PUB); } else if (ReceiveMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) { ReceiveMessageRequest request = (ReceiveMessageRequest) messageV3; accessResource.addGroupResourceAndPerm(request.getGroup(), Permission.SUB); diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessResourceTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessResourceTest.java index 8ff3d610486..bccd37e39ef 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessResourceTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessResourceTest.java @@ -19,10 +19,15 @@ import java.util.HashMap; import java.util.Map; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.Resource; +import com.google.protobuf.GeneratedMessageV3; +import org.apache.rocketmq.acl.common.AuthenticationHeader; import org.apache.rocketmq.acl.common.Permission; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; import org.junit.Assert; @@ -33,6 +38,8 @@ public class PlainAccessResourceTest { public static final String DEFAULT_PRODUCER_GROUP = "PID_acl"; public static final String DEFAULT_CONSUMER_GROUP = "GID_acl"; public static final String DEFAULT_REMOTE_ADDR = "192.128.1.1"; + public static final String AUTH_HEADER = + "Signature Credential=1234567890/test, SignedHeaders=host, Signature=1234567890"; @Test public void testParseSendNormal() { @@ -93,4 +100,34 @@ public void testParseSendRetryV2() { Assert.assertEquals(permMap, accessResource.getResourcePermMap()); } + + @Test + public void testParseRecallMessage() { + // remoting + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setTopic(DEFAULT_TOPIC); + requestHeader.setProducerGroup(DEFAULT_PRODUCER_GROUP); + requestHeader.setRecallHandle("handle"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + request.makeCustomHeaderToNet(); + + PlainAccessResource accessResource = PlainAccessResource.parse(request, DEFAULT_REMOTE_ADDR); + Assert.assertTrue(Permission.PUB == accessResource.getResourcePermMap().get(DEFAULT_TOPIC)); + + // grpc + GeneratedMessageV3 grpcRequest = RecallMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(DEFAULT_TOPIC).build()) + .setRecallHandle("handle") + .build(); + accessResource = PlainAccessResource.parse(grpcRequest, mockAuthenticationHeader()); + Assert.assertTrue(Permission.PUB == accessResource.getResourcePermMap().get(DEFAULT_TOPIC)); + } + + private AuthenticationHeader mockAuthenticationHeader() { + return AuthenticationHeader.builder() + .remoteAddress(DEFAULT_REMOTE_ADDR) + .authorization(AUTH_HEADER) + .datetime("datetime") + .build(); + } } diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java index e69abdaf805..bf86892ea61 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java @@ -25,6 +25,7 @@ import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.RecallMessageRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.SendMessageRequest; import apache.rocketmq.v2.Subscription; @@ -101,6 +102,10 @@ public List build(Metadata metadata, GeneratedMessa } result = newPubContext(metadata, request.getMessages(0).getTopic()); } + if (message instanceof RecallMessageRequest) { + RecallMessageRequest request = (RecallMessageRequest) message; + result = newPubContext(metadata, request.getTopic()); + } if (message instanceof EndTransactionRequest) { EndTransactionRequest request = (EndTransactionRequest) message; result = newPubContext(metadata, request.getTopic()); @@ -207,6 +212,10 @@ public List build(ChannelHandlerContext context, Re result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp)); } break; + case RequestCode.RECALL_MESSAGE: + topic = Resource.ofTopic(fields.get(TOPIC)); + result.add(DefaultAuthorizationContext.of(subject, topic, Action.PUB, sourceIp)); + break; case RequestCode.END_TRANSACTION: if (StringUtils.isNotBlank(fields.get(TOPIC))) { topic = Resource.ofTopic(fields.get(TOPIC)); diff --git a/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java b/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java index 4ee73f3d797..c73e07d7529 100644 --- a/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java +++ b/auth/src/test/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilderTest.java @@ -28,6 +28,7 @@ import apache.rocketmq.v2.Publishing; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.RecallMessageRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.Resource; import apache.rocketmq.v2.SendMessageRequest; @@ -65,6 +66,7 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.remoting.protocol.header.UnregisterClientRequestHeader; @@ -122,6 +124,19 @@ public void buildGrpc() { Assert.assertEquals(result.get(0).getChannelId(), "channel-id"); Assert.assertEquals(result.get(0).getRpcCode(), SendMessageRequest.getDescriptor().getFullName()); + request = RecallMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName("topic").build()) + .setRecallHandle("handle") + .build(); + result = builder.build(metadata, request); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(result.get(0).getSubject().getSubjectKey(), "User:rocketmq"); + Assert.assertEquals(result.get(0).getResource().getResourceKey(), "Topic:topic"); + Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB))); + Assert.assertEquals(result.get(0).getSourceIp(), "192.168.0.1"); + Assert.assertEquals(result.get(0).getChannelId(), "channel-id"); + Assert.assertEquals(result.get(0).getRpcCode(), RecallMessageRequest.getDescriptor().getFullName()); + request = EndTransactionRequest.newBuilder() .setTopic(Resource.newBuilder().setName("topic").build()) .build(); @@ -315,6 +330,22 @@ public void buildRemoting() { Assert.assertEquals("Group:group", result.get(0).getResource().getResourceKey()); Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.SUB))); + RecallMessageRequestHeader recallMessageRequestHeader = new RecallMessageRequestHeader(); + recallMessageRequestHeader.setTopic("topic"); + recallMessageRequestHeader.setRecallHandle("handle"); + request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, recallMessageRequestHeader); + request.setVersion(441); + request.addExtField("AccessKey", "rocketmq"); + request.makeCustomHeaderToNet(); + result = builder.build(channelHandlerContext, request); + Assert.assertEquals(1, result.size()); + Assert.assertEquals("User:rocketmq", result.get(0).getSubject().getSubjectKey()); + Assert.assertEquals("Topic:topic", result.get(0).getResource().getResourceKey()); + Assert.assertTrue(result.get(0).getActions().containsAll(Arrays.asList(Action.PUB))); + Assert.assertEquals("192.168.0.1", result.get(0).getSourceIp()); + Assert.assertEquals("channel-id", result.get(0).getChannelId()); + Assert.assertEquals(RequestCode.RECALL_MESSAGE + "", result.get(0).getRpcCode()); + EndTransactionRequestHeader endTransactionRequestHeader = new EndTransactionRequestHeader(); endTransactionRequestHeader.setTopic("topic"); request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, endTransactionRequestHeader); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index e1edd2f5126..744aba19118 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -93,6 +93,7 @@ import org.apache.rocketmq.broker.processor.PullMessageProcessor; import org.apache.rocketmq.broker.processor.QueryAssignmentProcessor; import org.apache.rocketmq.broker.processor.QueryMessageProcessor; +import org.apache.rocketmq.broker.processor.RecallMessageProcessor; import org.apache.rocketmq.broker.processor.ReplyMessageProcessor; import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.broker.schedule.ScheduleMessageService; @@ -210,6 +211,7 @@ public class BrokerController { protected final QueryAssignmentProcessor queryAssignmentProcessor; protected final ClientManageProcessor clientManageProcessor; protected final SendMessageProcessor sendMessageProcessor; + protected final RecallMessageProcessor recallMessageProcessor; protected final ReplyMessageProcessor replyMessageProcessor; protected final PullRequestHoldService pullRequestHoldService; protected final MessageArrivingListener messageArrivingListener; @@ -369,6 +371,7 @@ public BrokerController( this.ackMessageProcessor = new AckMessageProcessor(this); this.changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(this); this.sendMessageProcessor = new SendMessageProcessor(this); + this.recallMessageProcessor = new RecallMessageProcessor(this); this.replyMessageProcessor = new ReplyMessageProcessor(this); this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService, this.popMessageProcessor, this.notificationProcessor); this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this); @@ -1096,10 +1099,12 @@ public void registerProcessor() { this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor); this.remotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendMessageProcessor, this.sendMessageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.SEND_BATCH_MESSAGE, sendMessageProcessor, this.sendMessageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendMessageProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.RECALL_MESSAGE, recallMessageProcessor, this.sendMessageExecutor); /** * PullMessageProcessor */ @@ -2424,6 +2429,10 @@ public SendMessageProcessor getSendMessageProcessor() { return sendMessageProcessor; } + public RecallMessageProcessor getRecallMessageProcessor() { + return recallMessageProcessor; + } + public QueryAssignmentProcessor getQueryAssignmentProcessor() { return queryAssignmentProcessor; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java new file mode 100644 index 00000000000..7a652f43151 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.producer.RecallMessageHandle; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.timer.TimerMessageStore; + +import java.nio.charset.StandardCharsets; + +public class RecallMessageProcessor implements NettyRequestProcessor { + private static final String RECALL_MESSAGE_TAG = "_RECALL_TAG_"; + private final BrokerController brokerController; + + public RecallMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws + RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class); + response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); + final RecallMessageRequestHeader requestHeader = + request.decodeCommandCustomHeader(RecallMessageRequestHeader.class); + + if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) { + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + response.setRemark("recall failed, broker service not available"); + return response; + } + + final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); + if (this.brokerController.getMessageStore().now() < startTimestamp) { + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("recall failed, broker service not available"); + return response; + } + + if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission()) + && !this.brokerController.getBrokerConfig().isAllowRecallWhenBrokerNotWriteable()) { + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("recall failed, broker service not available"); + return response; + } + + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("recall failed, the topic[" + requestHeader.getTopic() + "] not exist"); + return response; + } + + RecallMessageHandle.HandleV1 handle; + try { + handle = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(requestHeader.getRecallHandle()); + } catch (DecoderException e) { + response.setCode(ResponseCode.ILLEGAL_OPERATION); + response.setRemark(e.getMessage()); + return response; + } + + if (!requestHeader.getTopic().equals(handle.getTopic())) { + response.setCode(ResponseCode.ILLEGAL_OPERATION); + response.setRemark("recall failed, topic not match"); + return response; + } + if (!brokerController.getBrokerConfig().getBrokerName().equals(handle.getBrokerName())) { + response.setCode(ResponseCode.ILLEGAL_OPERATION); + response.setRemark("recall failed, broker service not available"); + return response; + } + + long timestamp = NumberUtils.toLong(handle.getTimestampStr(), -1); + long timeLeft = timestamp - System.currentTimeMillis(); + if (timeLeft <= 0 + || timeLeft >= brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L) { + response.setCode(ResponseCode.ILLEGAL_OPERATION); + response.setRemark("recall failed, timestamp invalid"); + return response; + } + + MessageExtBrokerInner msgInner = buildMessage(ctx, requestHeader, handle); + long beginTimeMillis = this.brokerController.getMessageStore().now(); + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + handlePutMessageResult(putMessageResult, request, response, msgInner, ctx, beginTimeMillis); + return response; + } + + public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessageRequestHeader requestHeader, + RecallMessageHandle.HandleV1 handle) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(handle.getTopic()); + msgInner.setBody("0".getBytes(StandardCharsets.UTF_8)); + msgInner.setTags(RECALL_MESSAGE_TAG); + msgInner.setTagsCode(RECALL_MESSAGE_TAG.hashCode()); + msgInner.setQueueId(0); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY, + TimerMessageStore.buildDeleteKey(handle.getTopic(), handle.getMessageId())); + MessageAccessor.putProperty(msgInner, + MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId()); + MessageAccessor.putProperty(msgInner, + MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(handle.getTimestampStr())); + MessageAccessor.putProperty(msgInner, + MessageConst.PROPERTY_BORN_TIMESTAMP, String.valueOf(System.currentTimeMillis())); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRACE_CONTEXT, ""); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_PRODUCER_GROUP, requestHeader.getProducerGroup()); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.brokerController.getStoreHost()); + return msgInner; + } + + public void handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand request, + RemotingCommand response, MessageExt message, ChannelHandlerContext ctx, long beginTimeMillis) { + if (null == putMessageResult) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("recall failed, execute error"); + return; + } + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + switch (putMessageResult.getPutMessageStatus()) { + case PUT_OK: + this.brokerController.getBrokerStatsManager().incTopicPutNums( + message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); // system timer topic + this.brokerController.getBrokerStatsManager().incTopicPutSize( + message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes()); + this.brokerController.getBrokerStatsManager().incBrokerPutNums( + message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum()); + this.brokerController.getBrokerStatsManager().incTopicPutLatency( + message.getTopic(), 0, (int) (this.brokerController.getMessageStore().now() - beginTimeMillis)); + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SUCCESS); + responseHeader.setMsgId(MessageClientIDSetter.getUniqID(message)); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("recall failed, execute error"); + break; + } + } + + @Override + public boolean rejectRequest() { + return false; + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index db5b22888dc..669cd5e6771 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.CleanupPolicyUtils; @@ -483,6 +484,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult responseHeader.setQueueId(queueIdInt); responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); responseHeader.setTransactionId(MessageClientIDSetter.getUniqID(msg)); + attachRecallHandle(request, msg, responseHeader); RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext); if (rewriteResult != null) { @@ -647,6 +649,21 @@ private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx, } } + public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader) { + if (RequestCode.SEND_BATCH_MESSAGE == request.getCode() + || RequestCode.CONSUMER_SEND_MSG_BACK == request.getCode()) { + return; + } + String timestampStr = msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS); + String realTopic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); + if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor + String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic, + brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg)); + responseHeader.setRecallHandle(recallHandle); + } + } + private String diskUtil() { double physicRatio = 100; String storePath; diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java new file mode 100644 index 00000000000..7bd260cc2c0 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/RecallMessageProcessorTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.producer.RecallMessageHandle; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class RecallMessageProcessorTest { + private static final String TOPIC = "topic"; + private static final String BROKER_NAME = "brokerName"; + + private RecallMessageProcessor recallMessageProcessor; + @Mock + private BrokerConfig brokerConfig; + @Mock + private BrokerController brokerController; + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private MessageStoreConfig messageStoreConfig; + @Mock + private TopicConfigManager topicConfigManager; + @Mock + private MessageStore messageStore; + @Mock + private BrokerStatsManager brokerStatsManager; + @Mock + private Channel channel; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); + when(brokerController.getMessageStore()).thenReturn(messageStore); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + when(brokerConfig.getBrokerName()).thenReturn(BROKER_NAME); + when(brokerController.getBrokerStatsManager()).thenReturn(brokerStatsManager); + when(handlerContext.channel()).thenReturn(channel); + recallMessageProcessor = new RecallMessageProcessor(brokerController); + } + + @Test + public void testBuildMessage() { + String timestampStr = String.valueOf(System.currentTimeMillis()); + String id = "id"; + RecallMessageHandle.HandleV1 handle = new RecallMessageHandle.HandleV1(TOPIC, "brokerName", timestampStr, id); + MessageExtBrokerInner msg = + recallMessageProcessor.buildMessage(handlerContext, new RecallMessageRequestHeader(), handle); + + Assert.assertEquals(TOPIC, msg.getTopic()); + Map properties = MessageDecoder.string2messageProperties(msg.getPropertiesString()); + Assert.assertEquals(timestampStr, properties.get(MessageConst.PROPERTY_TIMER_DELIVER_MS)); + Assert.assertEquals(id, properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); + Assert.assertEquals(TOPIC + "+" + id, properties.get(MessageConst.PROPERTY_TIMER_DEL_UNIQKEY)); + } + + @Test + public void testHandlePutMessageResult() { + MessageExt message = new MessageExt(); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, "id"); + RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class); + recallMessageProcessor.handlePutMessageResult(null, null, response, message, handlerContext, 0L); + Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + + List okStatus = Arrays.asList(PutMessageStatus.PUT_OK, PutMessageStatus.FLUSH_DISK_TIMEOUT, + PutMessageStatus.FLUSH_SLAVE_TIMEOUT, PutMessageStatus.SLAVE_NOT_AVAILABLE); + + for (PutMessageStatus status : PutMessageStatus.values()) { + PutMessageResult putMessageResult = + new PutMessageResult(status, new AppendMessageResult(AppendMessageStatus.PUT_OK)); + recallMessageProcessor.handlePutMessageResult(putMessageResult, null, response, message, handlerContext, 0L); + if (okStatus.contains(status)) { + Assert.assertEquals(ResponseCode.SUCCESS, response.getCode()); + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + Assert.assertEquals("id", responseHeader.getMsgId()); + } else { + Assert.assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + } + } + } + + @Test + public void testProcessRequest_invalidStatus() throws RemotingCommandException { + RemotingCommand request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME); + RemotingCommand response; + + // role slave + when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.SLAVE_NOT_AVAILABLE, response.getCode()); + + // not reach startTimestamp + when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SYNC_MASTER); + when(messageStore.now()).thenReturn(0L); + when(brokerConfig.getStartAcceptSendRequestTimeStamp()).thenReturn(System.currentTimeMillis()); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode()); + } + + @Test + public void testProcessRequest_notWriteable() throws RemotingCommandException { + when(brokerConfig.getBrokerPermission()).thenReturn(4); + when(brokerConfig.isAllowRecallWhenBrokerNotWriteable()).thenReturn(false); + RemotingCommand request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME); + RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, response.getCode()); + } + + @Test + public void testProcessRequest_topicNotFound_or_notMatch() throws RemotingCommandException { + when(brokerConfig.getBrokerPermission()).thenReturn(6); + RemotingCommand request; + RemotingCommand response; + + // not found + request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode()); + + // not match + when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC)); + request = mockRequest(0, TOPIC, "anotherTopic", "id", BROKER_NAME); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode()); + } + + @Test + public void testProcessRequest_brokerNameNotMatch() throws RemotingCommandException { + when(brokerConfig.getBrokerPermission()).thenReturn(6); + when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC)); + + RemotingCommand request = mockRequest(0, TOPIC, "anotherTopic", "id", BROKER_NAME + "_other"); + RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode()); + } + + @Test + public void testProcessRequest_timestampInvalid() throws RemotingCommandException { + when(brokerConfig.getBrokerPermission()).thenReturn(6); + when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC)); + RemotingCommand request; + RemotingCommand response; + + // past timestamp + request = mockRequest(0, TOPIC, TOPIC, "id", BROKER_NAME); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode()); + + // timestamp overflow + when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400); + request = mockRequest(System.currentTimeMillis() + 86400 * 2 * 1000, TOPIC, TOPIC, "id", BROKER_NAME); + response = recallMessageProcessor.processRequest(handlerContext, request); + Assert.assertEquals(ResponseCode.ILLEGAL_OPERATION, response.getCode()); + } + + @Test + public void testProcessRequest_success() throws RemotingCommandException { + when(brokerConfig.getBrokerPermission()).thenReturn(6); + when(topicConfigManager.selectTopicConfig(TOPIC)).thenReturn(new TopicConfig(TOPIC)); + when(messageStoreConfig.getTimerMaxDelaySec()).thenReturn(86400); + when(messageStore.putMessage(any())).thenReturn( + new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + + String msgId = "msgId"; + RemotingCommand request = mockRequest(System.currentTimeMillis() + 90 * 1000, TOPIC, TOPIC, msgId, BROKER_NAME); + RemotingCommand response = recallMessageProcessor.processRequest(handlerContext, request); + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + Assert.assertEquals(ResponseCode.SUCCESS, response.getCode()); + Assert.assertEquals(msgId, responseHeader.getMsgId()); + verify(messageStore, times(1)).putMessage(any()); + } + + private RemotingCommand mockRequest(long timestamp, String requestTopic, String handleTopic, + String msgId, String brokerName) { + String handle = + RecallMessageHandle.HandleV1.buildHandle(handleTopic, brokerName, String.valueOf(timestamp), msgId); + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup("group"); + requestHeader.setTopic(requestTopic); + requestHeader.setRecallHandle(handle); + requestHeader.setBrokerName(brokerName); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index 442794dcd26..9da6a96ec99 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; +import org.apache.commons.codec.DecoderException; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.AbortProcessException; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; @@ -36,10 +37,13 @@ import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -50,12 +54,14 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; import org.apache.rocketmq.store.AppendMessageResult; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -69,6 +75,8 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.times; @RunWith(MockitoJUnitRunner.class) public class SendMessageProcessorTest { @@ -78,6 +86,8 @@ public class SendMessageProcessorTest { @Mock private Channel channel; @Spy + private BrokerConfig brokerConfig; + @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); @Mock @@ -98,6 +108,7 @@ public void init() { when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager); when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); when(brokerController.getPutMessageFutureExecutor()).thenReturn(Executors.newSingleThreadExecutor()); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); when(messageStore.now()).thenReturn(System.currentTimeMillis()); when(channel.remoteAddress()).thenReturn(new InetSocketAddress(1024)); when(handlerContext.channel()).thenReturn(channel); @@ -299,6 +310,60 @@ public void consumeMessageAfter(ConsumeMessageContext context) { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } + @Test + public void testAttachRecallHandle_skip() { + MessageExt message = new MessageExt(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_BATCH_MESSAGE, null); + sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader()); + + request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, null); + sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader()); + + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, null); + sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader()); + + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null); + sendMessageProcessor.attachRecallHandle(request, message, new SendMessageResponseHeader()); + + verify(brokerConfig, times(0)).getBrokerName(); + } + + @Test + public void testAttachRecallHandle_doAttach() throws DecoderException { + int[] precisionSet = {100, 200, 500, 1000}; + SendMessageResponseHeader responseHeader = new SendMessageResponseHeader(); + String id = MessageClientIDSetter.createUniqID(); + long timestamp = System.currentTimeMillis(); + + for (int precisionMs : precisionSet) { + long deliverMs = floor(timestamp, precisionMs); + MessageExt message = new MessageExt(); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, id); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_TIMER_OUT_MS, String.valueOf(deliverMs)); + MessageAccessor.putProperty(message, MessageConst.PROPERTY_REAL_TOPIC, topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, null); + sendMessageProcessor.attachRecallHandle(request, message, responseHeader); + Assert.assertNotNull(responseHeader.getRecallHandle()); + RecallMessageHandle.HandleV1 v1 = + (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(responseHeader.getRecallHandle()); + Assert.assertEquals(id, v1.getMessageId()); + Assert.assertEquals(topic, v1.getTopic()); + Assert.assertEquals(deliverMs + 1, Long.parseLong(v1.getTimestampStr())); + Assert.assertEquals(deliverMs, floor(Long.valueOf(v1.getTimestampStr()), precisionMs)); + } + } + + private long floor(long deliverMs, int precisionMs) { + assert precisionMs > 0; + if (deliverMs % precisionMs == 0) { + deliverMs -= precisionMs; + } else { + deliverMs = deliverMs / precisionMs * precisionMs; + } + return deliverMs; + } + private RemotingCommand createSendTransactionMsgCommand(int requestCode) { SendMessageRequestHeader header = createSendMsgRequestHeader(); int sysFlag = header.getSysFlag(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 554b1efa524..2e088ac9da5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -204,6 +204,8 @@ import org.apache.rocketmq.remoting.protocol.header.QuerySubscriptionByConsumerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.RemoveBrokerRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ResetMasterFlushOffsetHeader; import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; @@ -853,6 +855,7 @@ protected SendResult processSendResponse( uniqMsgId, responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); + sendResult.setRecallHandle(responseHeader.getRecallHandle()); String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); if (regionId == null || regionId.isEmpty()) { regionId = MixAll.DEFAULT_TRACE_REGION_ID; @@ -3525,4 +3528,49 @@ public List listAcl(String addr, String subjectFilter, String resourceF } throw new MQBrokerException(response.getCode(), response.getRemark()); } + + public String recallMessage( + final String addr, + RecallMessageRequestHeader requestHeader, + final long timeoutMillis + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + RecallMessageResponseHeader responseHeader = + response.decodeCommandCustomHeader(RecallMessageResponseHeader.class); + return responseHeader.getMsgId(); + } + default: + break; + } + throw new MQBrokerException(response.getCode(), response.getRemark(), addr); + } + + public void recallMessageAsync( + final String addr, + final RecallMessageRequestHeader requestHeader, + final long timeoutMillis, + final InvokeCallback invokeCallback + ) throws RemotingException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } + + @Override + public void operationSucceed(RemotingCommand response) { + invokeCallback.operationSucceed(response); + } + + @Override + public void operationFail(Throwable throwable) { + invokeCallback.operationFail(throwable); + } + }); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java index 0e2092b8a0f..6624b3100d8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java @@ -74,6 +74,8 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SearchOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; @@ -624,6 +626,26 @@ public CompletableFuture notification(String brokerAddr, NotificationRe }); } + public CompletableFuture recallMessageAsync(String brokerAddr, + RecallMessageRequestHeader requestHeader, long timeoutMillis) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture future = new CompletableFuture<>(); + if (ResponseCode.SUCCESS == response.getCode()) { + try { + RecallMessageResponseHeader responseHeader = + response.decodeCommandCustomHeader(RecallMessageResponseHeader.class); + future.complete(responseHeader.getMsgId()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } else { + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); + } + return future; + }); + } + public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { return getRemotingClient().invoke(brokerAddr, request, timeoutMillis); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 3d4fdbec373..15264f0e503 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -35,6 +35,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -81,6 +82,7 @@ import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageType; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.utils.CorrelationIdUtil; import org.apache.rocketmq.remoting.RPCHook; @@ -91,6 +93,7 @@ import org.apache.rocketmq.remoting.protocol.NamespaceUtil; import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -1549,6 +1552,40 @@ public void endTransaction( this.defaultMQProducer.getSendMsgTimeout()); } + public String recallMessage( + String topic, + String recallHandle) throws RemotingException, MQClientException, MQBrokerException, InterruptedException { + makeSureStateOK(); + Validators.checkTopic(topic); + if (NamespaceUtil.isRetryTopic(topic) || NamespaceUtil.isDLQTopic(topic)) { + throw new MQClientException("topic is not supported", null); + } + RecallMessageHandle.HandleV1 handleEntity; + try { + handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle); + } catch (Exception e) { + throw new MQClientException(e.getMessage(), null); + } + + tryToFindTopicPublishInfo(topic); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(handleEntity.getBrokerName()); + brokerAddr = StringUtils.isNotEmpty(brokerAddr) ? + // find another address to support multi proxy endpoints, + // may cause failure request in proxy-less mode when the broker is temporarily unavailable + brokerAddr : this.mQClientFactory.findBrokerAddrByTopic(topic); + if (StringUtils.isEmpty(brokerAddr)) { + log.warn("can't find broker service address. {}", handleEntity.getBrokerName()); + throw new MQClientException("The broker service address not found", null); + } + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTopic(topic); + requestHeader.setRecallHandle(recallHandle); + requestHeader.setBrokerName(handleEntity.getBrokerName()); + return this.mQClientFactory.getMQClientAPIImpl().recallMessage(brokerAddr, + requestHeader, this.defaultMQProducer.getSendMsgTimeout()); + } + public void setCallbackExecutor(final ExecutorService callbackExecutor) { this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index a8bf7cee85f..e3f81ad9685 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.lock.ReadWriteCASLock; +import org.apache.rocketmq.client.trace.hook.DefaultRecallMessageTraceHook; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.hook.EndTransactionTraceHookImpl; @@ -381,6 +382,8 @@ public void start() throws MQClientException { new SendMessageTraceHookImpl(traceDispatcher)); this.defaultMQProducerImpl.registerEndTransactionHook( new EndTransactionTraceHookImpl(traceDispatcher)); + this.defaultMQProducerImpl.getMqClientFactory().getMQClientAPIImpl().getRemotingClient() + .registerRPCHook(new DefaultRecallMessageTraceHook(traceDispatcher)); } catch (Throwable e) { logger.error("system mqtrace hook init failed ,maybe can't send msg trace data"); } @@ -1128,6 +1131,12 @@ public void send(Collection msgs, MessageQueue mq, this.defaultMQProducerImpl.send(batch(msgs), queueWithNamespace(mq), sendCallback, timeout); } + @Override + public String recallMessage(String topic, String recallHandle) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.recallMessage(withNamespace(topic), recallHandle); + } + /** * Sets an Executor to be used for executing callback methods. * diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index 8bd30e98d7b..4286fdd7f96 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -113,6 +113,9 @@ void send(final Collection msgs, final MessageQueue mq, final SendCallb final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + String recallMessage(String topic, String recallHandle) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + //for rpc Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java index dd7ea1cdc5f..d160eb4eae9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java @@ -29,6 +29,7 @@ public class SendResult { private String regionId; private boolean traceOn = true; private byte[] rawRespBody; + private String recallHandle; public SendResult() { } @@ -126,10 +127,18 @@ public void setOffsetMsgId(String offsetMsgId) { this.offsetMsgId = offsetMsgId; } + public String getRecallHandle() { + return recallHandle; + } + + public void setRecallHandle(String recallHandle) { + this.recallHandle = recallHandle; + } + @Override public String toString() { return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue - + ", queueOffset=" + queueOffset + "]"; + + ", queueOffset=" + queueOffset + ", recallHandle=" + recallHandle + "]"; } public void setRawRespBody(byte[] body) { diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java index 57e9b6410db..1e66aa0498d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -132,6 +132,19 @@ public static List decoderFromTraceDataString(String traceData) { endTransactionContext.setTraceBeans(new ArrayList<>(1)); endTransactionContext.getTraceBeans().add(bean); resList.add(endTransactionContext); + } else if (line[0].equals(TraceType.Recall.name())) { + TraceContext recallContext = new TraceContext(); + recallContext.setTraceType(TraceType.Recall); + recallContext.setTimeStamp(Long.parseLong(line[1])); + recallContext.setRegionId(line[2]); + recallContext.setGroupName(line[3]); + TraceBean bean = new TraceBean(); + bean.setTopic(line[4]); + bean.setMsgId(line[5]); + recallContext.setSuccess(Boolean.parseBoolean(line[6])); + recallContext.setTraceBeans(new ArrayList<>(1)); + recallContext.getTraceBeans().add(bean); + resList.add(recallContext); } } return resList; @@ -217,6 +230,17 @@ public static TraceTransferBean encoderFromContextBean(TraceContext ctx) { .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR); } break; + case Recall: { + TraceBean bean = ctx.getTraceBeans().get(0); + sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR) + .append(ctx.getTimeStamp()).append(TraceConstants.CONTENT_SPLITOR) + .append(ctx.getRegionId()).append(TraceConstants.CONTENT_SPLITOR) + .append(ctx.getGroupName()).append(TraceConstants.CONTENT_SPLITOR) + .append(bean.getTopic()).append(TraceConstants.CONTENT_SPLITOR) + .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR) + .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);// + } + break; default: } transferBean.setTransData(sb.toString()); diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java index 8870ddcbdb3..4c0e7d8ab26 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceType.java @@ -18,6 +18,7 @@ public enum TraceType { Pub, + Recall, SubBefore, SubAfter, EndTransaction, diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java new file mode 100644 index 00000000000..c490a7b3599 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/DefaultRecallMessageTraceHook.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.trace.hook; + +import org.apache.rocketmq.client.trace.TraceBean; +import org.apache.rocketmq.client.trace.TraceContext; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.TraceType; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.producer.RecallMessageHandle; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.NamespaceUtil; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; + +import java.util.ArrayList; + +public class DefaultRecallMessageTraceHook implements RPCHook { + + private static final String RECALL_TRACE_ENABLE_KEY = "com.rocketmq.recall.default.trace.enable"; + private boolean enableDefaultTrace = Boolean.parseBoolean(System.getProperty(RECALL_TRACE_ENABLE_KEY, "false")); + private TraceDispatcher traceDispatcher; + + public DefaultRecallMessageTraceHook(TraceDispatcher traceDispatcher) { + this.traceDispatcher = traceDispatcher; + } + + @Override + public void doBeforeRequest(String remoteAddr, RemotingCommand request) { + } + + @Override + public void doAfterResponse(String remoteAddr, RemotingCommand request, RemotingCommand response) { + if (request.getCode() != RequestCode.RECALL_MESSAGE + || !enableDefaultTrace + || null == response.getExtFields() + || null == response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION) + || null == traceDispatcher) { + return; + } + + try { + String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); + RecallMessageRequestHeader requestHeader = + request.decodeCommandCustomHeader(RecallMessageRequestHeader.class); + String topic = NamespaceUtil.withoutNamespace(requestHeader.getTopic()); + String group = NamespaceUtil.withoutNamespace(requestHeader.getProducerGroup()); + String recallHandle = requestHeader.getRecallHandle(); + RecallMessageHandle.HandleV1 handleV1 = + (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle); + + TraceBean traceBean = new TraceBean(); + traceBean.setTopic(topic); + traceBean.setMsgId(handleV1.getMessageId()); + + TraceContext traceContext = new TraceContext(); + traceContext.setRegionId(regionId); + traceContext.setTraceBeans(new ArrayList<>(1)); + traceContext.setTraceType(TraceType.Recall); + traceContext.setGroupName(group); + traceContext.getTraceBeans().add(traceBean); + traceContext.setSuccess(ResponseCode.SUCCESS == response.getCode()); + + traceDispatcher.append(traceContext); + } catch (Exception e) { + } + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index 81dc5883fb8..c76d0c734a0 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -118,6 +119,8 @@ import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SearchOffsetResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; @@ -142,6 +145,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.assertj.core.api.Assertions; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -170,6 +174,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -2028,6 +2033,77 @@ public void assertListAcl() throws RemotingException, InterruptedException, MQBr assertEquals(1, actual.get(0).getPolicies().size()); } + @Test + public void testRecallMessage() throws RemotingException, InterruptedException, MQBrokerException { + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setRecallHandle("handle"); + requestHeader.setBrokerName(brokerName); + + // success + mockInvokeSync(); + String msgId = MessageClientIDSetter.createUniqID(); + RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader(); + responseHeader.setMsgId(msgId); + setResponseHeader(responseHeader); + String result = mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout); + assertEquals(msgId, result); + + // error + when(response.getCode()).thenReturn(ResponseCode.SYSTEM_ERROR); + when(response.getRemark()).thenReturn("error"); + MQBrokerException e = assertThrows(MQBrokerException.class, () -> { + mqClientAPI.recallMessage(defaultBrokerAddr, requestHeader, defaultTimeout); + }); + assertEquals(ResponseCode.SYSTEM_ERROR, e.getResponseCode()); + assertEquals("error", e.getErrorMessage()); + assertEquals(defaultBrokerAddr, e.getBrokerAddr()); + } + + @Test + public void testRecallMessageAsync() throws RemotingException, InterruptedException { + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setRecallHandle("handle"); + requestHeader.setBrokerName(brokerName); + String msgId = "msgId"; + doAnswer((Answer) mock -> { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + responseHeader.setMsgId(msgId); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(response); + callback.operationSucceed(responseFuture.getResponseCommand()); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + + final CountDownLatch done = new CountDownLatch(1); + mqClientAPI.recallMessageAsync(defaultBrokerAddr, requestHeader, + defaultTimeout, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } + + @Override + public void operationSucceed(RemotingCommand response) { + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + Assert.assertEquals(msgId, responseHeader.getMsgId()); + done.countDown(); + } + + @Override + public void operationFail(Throwable throwable) { + } + }); + done.await(); + } + private Properties createProperties() { Properties result = new Properties(); result.put("key", "value"); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java index 6f692dff950..e2a29c9a21f 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExtTest.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.client.impl.mqclient; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; @@ -29,8 +30,11 @@ import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -108,4 +112,47 @@ public void testUpdateConsumerOffsetAsync_Fail() throws InterruptedException { assertEquals(customEx.getErrorMessage(), "QueueId is null, topic is testTopic"); } } + + @Test + public void testRecallMessageAsync_success() { + String msgId = "msgId"; + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup("group"); + requestHeader.setTopic("topic"); + requestHeader.setRecallHandle("handle"); + requestHeader.setBrokerName("brokerName"); + + RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader(); + responseHeader.setMsgId(msgId); + response.makeCustomHeaderToNet(); + CompletableFuture remotingFuture = new CompletableFuture<>(); + remotingFuture.complete(response); + doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + String resultId = + mqClientAPIExt.recallMessageAsync("brokerAddr", requestHeader, 3000L).join(); + Assert.assertEquals(msgId, resultId); + } + + @Test + public void testRecallMessageAsync_fail() { + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup("group"); + requestHeader.setTopic("topic"); + requestHeader.setRecallHandle("handle"); + requestHeader.setBrokerName("brokerName"); + + CompletableFuture remotingFuture = new CompletableFuture<>(); + remotingFuture.complete(RemotingCommand.createResponseCommand(ResponseCode.SERVICE_NOT_AVAILABLE, "")); + doReturn(remotingFuture).when(remotingClientMock).invoke(anyString(), any(RemotingCommand.class), anyLong()); + + CompletionException exception = Assert.assertThrows(CompletionException.class, () -> { + mqClientAPIExt.recallMessageAsync("brokerAddr", requestHeader, 3000L).join(); + }); + Assert.assertTrue(exception.getCause() instanceof MQBrokerException); + MQBrokerException cause = (MQBrokerException) exception.getCause(); + Assert.assertEquals(ResponseCode.SERVICE_NOT_AVAILABLE, cause.getResponseCode()); + } } \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java index a17fe43f461..77a83af19c0 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/selector/DefaultMQProducerImplTest.java @@ -33,11 +33,13 @@ import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.client.producer.TransactionMQProducer; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; import org.junit.Before; @@ -61,9 +63,11 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThrows; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -86,10 +90,15 @@ public class DefaultMQProducerImplTest { @Mock private MQClientInstance mQClientFactory; + @Mock + private MQClientAPIImpl mQClientAPIImpl; + private DefaultMQProducerImpl defaultMQProducerImpl; private final long defaultTimeout = 30000L; + private final String defaultBrokerName = "broker-0"; + private final String defaultBrokerAddr = "127.0.0.1:10911"; private final String defaultTopic = "testTopic"; @@ -104,7 +113,6 @@ public void init() throws Exception { when(clientConfig.queueWithNamespace(any())).thenReturn(messageQueue); when(mQClientFactory.getClientConfig()).thenReturn(clientConfig); when(mQClientFactory.getTopicRouteTable()).thenReturn(mock(ConcurrentMap.class)); - MQClientAPIImpl mQClientAPIImpl = mock(MQClientAPIImpl.class); when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mQClientAPIImpl); when(mQClientFactory.findBrokerAddressInPublish(or(isNull(), anyString()))).thenReturn(defaultBrokerAddr); when(message.getTopic()).thenReturn(defaultTopic); @@ -313,6 +321,39 @@ public void assertCheckListener() { assertNull(defaultMQProducerImpl.checkListener()); } + @Test + public void testRecallMessage_invalid() { + assertThrows(MQClientException.class, () -> { + defaultMQProducerImpl.recallMessage(MixAll.REPLY_TOPIC_POSTFIX + defaultTopic, "handle"); + }); + assertThrows(MQClientException.class, () -> { + defaultMQProducerImpl.recallMessage(MixAll.DLQ_GROUP_TOPIC_PREFIX + defaultTopic, "handle"); + }); + assertThrows(MQClientException.class, () -> { + defaultMQProducerImpl.recallMessage(defaultTopic, "handle"); + }); + } + + @Test + public void testRecallMessage_addressNotFound() { + String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, "1", "id"); + when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(null); + MQClientException e = assertThrows(MQClientException.class, () -> { + defaultMQProducerImpl.recallMessage(defaultTopic, handle); + }); + assertEquals("The broker service address not found", e.getErrorMessage()); + } + + @Test + public void testRecallMessage_success() + throws RemotingException, MQClientException, MQBrokerException, InterruptedException { + String handle = RecallMessageHandle.HandleV1.buildHandle(defaultTopic, defaultBrokerName, "1", "id"); + when(mQClientFactory.findBrokerAddressInPublish(defaultBrokerName)).thenReturn(defaultBrokerAddr); + when(mQClientAPIImpl.recallMessage(any(), any(), anyLong())).thenReturn("id"); + String result = defaultMQProducerImpl.recallMessage(defaultTopic, handle); + assertEquals("id", result); + } + private void setMQClientFactory() throws IllegalAccessException, NoSuchFieldException { setField(defaultMQProducerImpl, "mQClientFactory", mQClientFactory); } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index c0b557dfa11..bac2e2c7e40 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -444,6 +444,8 @@ public class BrokerConfig extends BrokerIdentity { */ private String configManagerVersion = ConfigManagerVersion.V1.getVersion(); + private boolean allowRecallWhenBrokerNotWriteable = true; + public String getConfigBlackList() { return configBlackList; } @@ -1932,4 +1934,12 @@ public String getConfigManagerVersion() { public void setConfigManagerVersion(String configManagerVersion) { this.configManagerVersion = configManagerVersion; } + + public boolean isAllowRecallWhenBrokerNotWriteable() { + return allowRecallWhenBrokerNotWriteable; + } + + public void setAllowRecallWhenBrokerNotWriteable(boolean allowRecallWhenBrokerNotWriteable) { + this.allowRecallWhenBrokerNotWriteable = allowRecallWhenBrokerNotWriteable; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java new file mode 100644 index 00000000000..b00b15bd863 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/producer/RecallMessageHandle.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.producer; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.lang3.StringUtils; + +import java.util.Base64; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * handle to recall a message, only support delay message for now + * v1 pattern like this: + * version topic brokerName timestamp messageId + * use Base64 to encode it + */ +public class RecallMessageHandle { + private static final String SEPARATOR = " "; + private static final String VERSION_1 = "v1"; + + public static class HandleV1 extends RecallMessageHandle { + private String version; + private String topic; + private String brokerName; + private String timestampStr; + private String messageId; // id of unique key + + public HandleV1(String topic, String brokerName, String timestamp, String messageId) { + this.version = VERSION_1; + this.topic = topic; + this.brokerName = brokerName; + this.timestampStr = timestamp; + this.messageId = messageId; + } + + // no param check + public static String buildHandle(String topic, String brokerName, String timestampStr, String messageId) { + String rawString = String.join(SEPARATOR, VERSION_1, topic, brokerName, timestampStr, messageId); + return Base64.getUrlEncoder().encodeToString(rawString.getBytes(UTF_8)); + } + + public String getTopic() { + return topic; + } + + public String getBrokerName() { + return brokerName; + } + + public String getTimestampStr() { + return timestampStr; + } + + public String getMessageId() { + return messageId; + } + + public String getVersion() { + return version; + } + } + + public static RecallMessageHandle decodeHandle(String handle) throws DecoderException { + if (StringUtils.isEmpty(handle)) { + throw new DecoderException("recall handle is invalid"); + } + String rawString; + try { + rawString = + new String(Base64.getUrlDecoder().decode(handle.getBytes(UTF_8)), UTF_8); + } catch (IllegalArgumentException e) { + throw new DecoderException("recall handle is invalid"); + } + String[] items = rawString.split(SEPARATOR); + if (!VERSION_1.equals(items[0]) || items.length < 5) { + throw new DecoderException("recall handle is invalid"); + } + return new HandleV1(items[1], items[2], items[3], items[4]); + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java b/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java new file mode 100644 index 00000000000..56608227693 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/producer/RecallMessageHandleTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.producer; + +import org.apache.commons.codec.DecoderException; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class RecallMessageHandleTest { + @Test + public void testHandleInvalid() { + Assert.assertThrows(DecoderException.class, () -> { + RecallMessageHandle.decodeHandle(""); + }); + Assert.assertThrows(DecoderException.class, () -> { + RecallMessageHandle.decodeHandle(null); + }); + + Assert.assertThrows(DecoderException.class, () -> { + String invalidHandle = Base64.getUrlEncoder().encodeToString("v1 a b c".getBytes(StandardCharsets.UTF_8)); + RecallMessageHandle.decodeHandle(invalidHandle); + }); + Assert.assertThrows(DecoderException.class, () -> { + String invalidHandle = Base64.getUrlEncoder().encodeToString("v2 a b c d".getBytes(StandardCharsets.UTF_8)); + RecallMessageHandle.decodeHandle(invalidHandle); + }); + Assert.assertThrows(DecoderException.class, () -> { + String invalidHandle = "v1 a b c d"; + RecallMessageHandle.decodeHandle(invalidHandle); + }); + } + + @Test + public void testEncodeAndDecodeV1() throws DecoderException { + String topic = "topic"; + String brokerName = "broker-0"; + String timestampStr = String.valueOf(System.currentTimeMillis()); + String messageId = MessageClientIDSetter.createUniqID(); + String handle = RecallMessageHandle.HandleV1.buildHandle(topic, brokerName, timestampStr, messageId); + RecallMessageHandle handleEntity = RecallMessageHandle.decodeHandle(handle); + Assert.assertTrue(handleEntity instanceof RecallMessageHandle.HandleV1); + RecallMessageHandle.HandleV1 handleV1 = (RecallMessageHandle.HandleV1) handleEntity; + Assert.assertEquals(handleV1.getVersion(), "v1"); + Assert.assertEquals(handleV1.getTopic(), topic); + Assert.assertEquals(handleV1.getBrokerName(), brokerName); + Assert.assertEquals(handleV1.getTimestampStr(), timestampStr); + Assert.assertEquals(handleV1.getMessageId(), messageId); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java index 866124d747c..f5edc03ba4a 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java @@ -25,6 +25,7 @@ import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.RecallMessageRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.SendMessageRequest; import java.util.HashMap; @@ -38,6 +39,7 @@ public class RequestMapping { put(QueryRouteRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC); put(HeartbeatRequest.getDescriptor().getFullName(), RequestCode.HEART_BEAT); put(SendMessageRequest.getDescriptor().getFullName(), RequestCode.SEND_MESSAGE_V2); + put(RecallMessageRequest.getDescriptor().getFullName(), RequestCode.RECALL_MESSAGE); put(QueryAssignmentRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC); put(ReceiveMessageRequest.getDescriptor().getFullName(), RequestCode.PULL_MESSAGE); put(AckMessageRequest.getDescriptor().getFullName(), RequestCode.UPDATE_CONSUMER_OFFSET); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java index 091e9086ecc..3c6f120ee58 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java @@ -32,6 +32,8 @@ import apache.rocketmq.v2.QueryAssignmentResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.SendMessageRequest; @@ -51,6 +53,7 @@ import org.apache.rocketmq.proxy.grpc.v2.consumer.ChangeInvisibleDurationActivity; import org.apache.rocketmq.proxy.grpc.v2.consumer.ReceiveMessageActivity; import org.apache.rocketmq.proxy.grpc.v2.producer.ForwardMessageToDLQActivity; +import org.apache.rocketmq.proxy.grpc.v2.producer.RecallMessageActivity; import org.apache.rocketmq.proxy.grpc.v2.producer.SendMessageActivity; import org.apache.rocketmq.proxy.grpc.v2.route.RouteActivity; import org.apache.rocketmq.proxy.grpc.v2.transaction.EndTransactionActivity; @@ -65,6 +68,7 @@ public class DefaultGrpcMessingActivity extends AbstractStartAndShutdown impleme protected AckMessageActivity ackMessageActivity; protected ChangeInvisibleDurationActivity changeInvisibleDurationActivity; protected SendMessageActivity sendMessageActivity; + protected RecallMessageActivity recallMessageActivity; protected ForwardMessageToDLQActivity forwardMessageToDLQActivity; protected EndTransactionActivity endTransactionActivity; protected RouteActivity routeActivity; @@ -82,6 +86,7 @@ protected void init(MessagingProcessor messagingProcessor) { this.ackMessageActivity = new AckMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.recallMessageActivity = new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); @@ -145,6 +150,12 @@ public CompletableFuture changeInvisibleDuratio return this.changeInvisibleDurationActivity.changeInvisibleDuration(ctx, request); } + @Override + public CompletableFuture recallMessage(ProxyContext ctx, + RecallMessageRequest request) { + return this.recallMessageActivity.recallMessage(ctx, request); + } + @Override public ContextStreamObserver telemetry(StreamObserver responseObserver) { return this.clientActivity.telemetry(responseObserver); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java index 4f029dec336..c470eda55ca 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java @@ -35,6 +35,8 @@ import apache.rocketmq.v2.QueryAssignmentResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.SendMessageRequest; @@ -371,6 +373,25 @@ public void changeInvisibleDuration(ChangeInvisibleDurationRequest request, } } + @Override + public void recallMessage(RecallMessageRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = + status -> RecallMessageResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + this.addExecutor(this.producerThreadPoolExecutor, // reuse producer thread pool + context, + request, + () -> grpcMessingActivity.recallMessage(context, request) + .whenComplete((response, throwable) -> + writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + @Override public StreamObserver telemetry(StreamObserver responseObserver) { Function statusResponseCreator = status -> TelemetryCommand.newBuilder().setStatus(status).build(); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java index 77bd3a88f9d..db15f25f6f7 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java @@ -33,6 +33,8 @@ import apache.rocketmq.v2.QueryAssignmentResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.SendMessageRequest; @@ -69,5 +71,7 @@ CompletableFuture notifyClientTermination(Proxy CompletableFuture changeInvisibleDuration(ProxyContext ctx, ChangeInvisibleDurationRequest request); + CompletableFuture recallMessage(ProxyContext ctx, RecallMessageRequest request); + ContextStreamObserver telemetry(StreamObserver responseObserver); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java new file mode 100644 index 00000000000..28ec97dca34 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivity.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; +import apache.rocketmq.v2.Resource; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +public class RecallMessageActivity extends AbstractMessingActivity { + + public RecallMessageActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + public CompletableFuture recallMessage(ProxyContext ctx, + RecallMessageRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + Resource topic = request.getTopic(); + validateTopic(topic); + + future = this.messagingProcessor.recallMessage( + ctx, + topic.getName(), + request.getRecallHandle(), + Duration.ofSeconds(2).toMillis() + ).thenApply(result -> RecallMessageResponse.newBuilder() + .setMessageId(result) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java index 8a3d315c68c..f7b8014bb99 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java @@ -341,6 +341,7 @@ protected SendMessageResponse convertToSendMessageResponse(ProxyContext ctx, Sen .setOffset(result.getQueueOffset()) .setMessageId(StringUtils.defaultString(result.getMsgId())) .setTransactionId(StringUtils.defaultString(result.getTransactionId())) + .setRecallHandle(StringUtils.defaultString(result.getRecallHandle())) .build(); break; default: diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java index 9c494d7a451..d0c0dd6e655 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java @@ -261,6 +261,12 @@ public CompletableFuture getMinOffset(ProxyContext ctx, MessageQueue messa return this.consumerProcessor.getMinOffset(ctx, messageQueue, timeoutMillis); } + @Override + public CompletableFuture recallMessage(ProxyContext ctx, String topic, + String recallHandle, long timeoutMillis) { + return this.producerProcessor.recallMessage(ctx, topic, recallHandle, timeoutMillis); + } + @Override public CompletableFuture request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java index 03d28262d73..fee0465e2bf 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java @@ -260,6 +260,13 @@ CompletableFuture getMinOffset( long timeoutMillis ); + CompletableFuture recallMessage( + ProxyContext ctx, + String topic, + String recallHandle, + long timeoutMillis + ); + CompletableFuture request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java index 4f2d5280d37..43e16ddd2d7 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import org.apache.commons.codec.DecoderException; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; @@ -32,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageId; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.FutureUtils; @@ -49,6 +51,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; public class ProducerProcessor extends AbstractProcessor { @@ -124,6 +127,33 @@ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSe return FutureUtils.addExecutor(future, this.executor); } + public CompletableFuture recallMessage(ProxyContext ctx, String topic, + String recallHandle, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) { + TopicMessageType messageType = serviceManager.getMetadataService().getTopicMessageType(ctx, topic); + topicMessageTypeValidator.validate(messageType, TopicMessageType.DELAY); + } + + RecallMessageHandle.HandleV1 handleEntity; + try { + handleEntity = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(recallHandle); + } catch (DecoderException e) { + throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } + String brokerName = handleEntity.getBrokerName(); + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setRecallHandle(recallHandle); + requestHeader.setBrokerName(brokerName); + future = serviceManager.getMessageService().recallMessage(ctx, brokerName, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + protected void fillTransactionData(ProxyContext ctx, String producerGroup, AddressableMessageQueue messageQueue, SendResult sendResult, List messageList) { try { MessageId id; diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java index 14c7c0db6fa..8c44305b42c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java @@ -45,6 +45,7 @@ import org.apache.rocketmq.proxy.remoting.activity.GetTopicRouteActivity; import org.apache.rocketmq.proxy.remoting.activity.PopMessageActivity; import org.apache.rocketmq.proxy.remoting.activity.PullMessageActivity; +import org.apache.rocketmq.proxy.remoting.activity.RecallMessageActivity; import org.apache.rocketmq.proxy.remoting.activity.SendMessageActivity; import org.apache.rocketmq.proxy.remoting.activity.TransactionActivity; import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager; @@ -75,6 +76,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu protected final ClientManagerActivity clientManagerActivity; protected final ConsumerManagerActivity consumerManagerActivity; protected final SendMessageActivity sendMessageActivity; + protected final RecallMessageActivity recallMessageActivity; protected final TransactionActivity transactionActivity; protected final PullMessageActivity pullMessageActivity; protected final PopMessageActivity popMessageActivity; @@ -97,6 +99,7 @@ public RemotingProtocolServer(MessagingProcessor messagingProcessor, List getMinOffset(ProxyContext ctx, AddressableMessage ); } + @Override + public CompletableFuture recallMessage(ProxyContext ctx, String brokerName, + RecallMessageRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().recallMessageAsync( + this.resolveBrokerAddr(ctx, brokerName), + requestHeader, + timeoutMillis + ); + } + @Override public CompletableFuture request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java index a8088a95d0a..cb9b7a4ae00 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -71,6 +71,8 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; @@ -153,6 +155,7 @@ public CompletableFuture> sendMessage(ProxyContext ctx, Address sendResult.setQueueOffset(responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); sendResult.setOffsetMsgId(responseHeader.getMsgId()); + sendResult.setRecallHandle(responseHeader.getRecallHandle()); return Collections.singletonList(sendResult); }); } @@ -470,6 +473,32 @@ public CompletableFuture getMinOffset(ProxyContext ctx, AddressableMessage throw new NotImplementedException("getMinOffset is not implemented in LocalMessageService"); } + @Override + public CompletableFuture recallMessage(ProxyContext ctx, String brokerName, + RecallMessageRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = + LocalRemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader, ctx.getLanguage()); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getRecallMessageProcessor() + .processRequest(channelHandlerContext, command); + future.complete(response); + } catch (Exception e) { + log.error("Fail to process recallMessage command", e); + future.completeExceptionally(e); + } + return future.thenApply(r -> { + switch (r.getCode()) { + case ResponseCode.SUCCESS: + return ((RecallMessageResponseHeader) r.readCustomHeader()).getMsgId(); + default: + throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark()); + } + }); + } + @Override public CompletableFuture request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java index 61accbc0412..80f5ae7217c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.UpdateConsumerOffsetRequestHeader; @@ -155,6 +156,13 @@ CompletableFuture getMinOffset( long timeoutMillis ); + CompletableFuture recallMessage( + ProxyContext ctx, + String brokerName, + RecallMessageRequestHeader requestHeader, + long timeoutMillis + ); + CompletableFuture request(ProxyContext ctx, String brokerName, RemotingCommand request, long timeoutMillis); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java new file mode 100644 index 00000000000..e42aeadbb6b --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/RecallMessageActivityTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; +import apache.rocketmq.v2.Resource; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.when; + +public class RecallMessageActivityTest extends BaseActivityTest { + private RecallMessageActivity recallMessageActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.recallMessageActivity = + new RecallMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testRecallMessage_success() { + when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong())) + .thenReturn(CompletableFuture.completedFuture("msgId")); + + RecallMessageResponse response = this.recallMessageActivity.recallMessage( + createContext(), + RecallMessageRequest.newBuilder() + .setRecallHandle("handle") + .setTopic(Resource.newBuilder().setResourceNamespace("ns").setName("topic")) + .build() + ).join(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals("msgId", response.getMessageId()); + } + + @Test + public void testRecallMessage_fail() { + CompletableFuture exceptionFuture = new CompletableFuture(); + when(this.messagingProcessor.recallMessage(any(), any(), any(), anyLong())).thenReturn(exceptionFuture); + exceptionFuture.completeExceptionally( + new ProxyException(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, "info")); + + CompletionException exception = Assert.assertThrows(CompletionException.class, () -> { + this.recallMessageActivity.recallMessage( + createContext(), + RecallMessageRequest.newBuilder() + .setRecallHandle("handle") + .setTopic(Resource.newBuilder().setResourceNamespace("ns").setName("topic")) + .build() + ).join(); + }); + Assert.assertTrue(exception.getCause() instanceof ProxyException); + ProxyException cause = (ProxyException) exception.getCause(); + Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode()); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java index 3192d5c8dfb..6729ef0c4b3 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.Executors; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; @@ -34,20 +35,25 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.producer.RecallMessageHandle; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; import org.apache.rocketmq.proxy.service.transaction.TransactionData; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.assertj.core.util.Lists; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -205,6 +211,39 @@ public void testForwardMessageToDeadLetterQueue() throws Throwable { assertEquals(CONSUMER_GROUP, requestHeader.getGroup()); } + @Test + public void testRecallMessage_notDelayMessage() { + when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.NORMAL); + CompletionException exception = Assert.assertThrows(CompletionException.class, () -> { + producerProcessor.recallMessage(createContext(), TOPIC, "handle", 3000).join(); + }); + assertTrue(exception.getCause() instanceof ProxyException); + ProxyException cause = (ProxyException) exception.getCause(); + assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, cause.getCode()); + } + + @Test + public void testRecallMessage_invalidRecallHandle() { + when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY); + CompletionException exception = Assert.assertThrows(CompletionException.class, () -> { + producerProcessor.recallMessage(createContext(), TOPIC, "handle", 3000).join(); + }); + assertTrue(exception.getCause() instanceof ProxyException); + ProxyException cause = (ProxyException) exception.getCause(); + assertEquals("recall handle is invalid", cause.getMessage()); + } + + @Test + public void testRecallMessage_success() { + when(metadataService.getTopicMessageType(any(), any())).thenReturn(TopicMessageType.DELAY); + when(this.messageService.recallMessage(any(), any(), any(), anyLong())) + .thenReturn(CompletableFuture.completedFuture("msgId")); + + String handle = RecallMessageHandle.HandleV1.buildHandle(TOPIC, "brokerName", "timestampStr", "whateverId"); + String msgId = producerProcessor.recallMessage(createContext(), TOPIC, handle, 3000).join(); + assertEquals("msgId", msgId); + } + private static String createOffsetMsgId(long commitLogOffset) { int msgIDLength = 4 + 4 + 8; ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java new file mode 100644 index 00000000000..7d64923d774 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/remoting/activity/RecallMessageActivityTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.remoting.activity; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.config.InitConfigTest; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.channel.SimpleChannel; +import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.concurrent.CompletableFuture; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class RecallMessageActivityTest extends InitConfigTest { + private static final String TOPIC = "topic"; + private static final String GROUP = "group"; + private static final String BROKER_NAME = "brokerName"; + + private RecallMessageActivity recallMessageActivity; + @Mock + private MessagingProcessor messagingProcessor; + @Mock + private MetadataService metadataService; + + @Spy + private ChannelHandlerContext ctx = new SimpleChannelHandlerContext(new SimpleChannel(null, "1", "2")) { + @Override + public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) { + return null; + } + }; + + @Before + public void init() { + recallMessageActivity = new RecallMessageActivity(null, messagingProcessor); + when(messagingProcessor.getMetadataService()).thenReturn(metadataService); + } + + @Test + public void testRecallMessage_notDelayMessage() { + when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.NORMAL); + ProxyException exception = Assert.assertThrows(ProxyException.class, () -> { + recallMessageActivity.processRequest0(ctx, mockRequest(), null); + }); + Assert.assertEquals(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, exception.getCode()); + } + + @Test + public void testRecallMessage_success() throws Exception { + when(metadataService.getTopicMessageType(any(), eq(TOPIC))).thenReturn(TopicMessageType.DELAY); + RemotingCommand request = mockRequest(); + RemotingCommand expectResponse = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""); + when(messagingProcessor.request(any(), eq(BROKER_NAME), eq(request), anyLong())) + .thenReturn(CompletableFuture.completedFuture(expectResponse)); + RemotingCommand response = recallMessageActivity.processRequest0(ctx, request, null); + Assert.assertNull(response); + verify(ctx, times(1)).writeAndFlush(eq(expectResponse)); + } + + private RemotingCommand mockRequest() { + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + requestHeader.setProducerGroup(GROUP); + requestHeader.setTopic(TOPIC); + requestHeader.setRecallHandle("handle"); + requestHeader.setBrokerName(BROKER_NAME); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.RECALL_MESSAGE, requestHeader); + request.makeCustomHeaderToNet(); + return request; + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java index f7a656d7682..20ce2a16848 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java @@ -26,12 +26,14 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.processor.AckMessageProcessor; import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor; import org.apache.rocketmq.broker.processor.EndTransactionProcessor; import org.apache.rocketmq.broker.processor.PopMessageProcessor; +import org.apache.rocketmq.broker.processor.RecallMessageProcessor; import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.AckStatus; @@ -68,8 +70,11 @@ import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.PopMessageResponseHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -94,6 +99,8 @@ public class LocalMessageServiceTest extends InitConfigTest { @Mock private AckMessageProcessor ackMessageProcessorMock; @Mock + private RecallMessageProcessor recallMessageProcessorMock; + @Mock private BrokerController brokerControllerMock; private ProxyContext proxyContext; @@ -122,6 +129,7 @@ public void setUp() throws Throwable { Mockito.when(brokerControllerMock.getChangeInvisibleTimeProcessor()).thenReturn(changeInvisibleTimeProcessorMock); Mockito.when(brokerControllerMock.getAckMessageProcessor()).thenReturn(ackMessageProcessorMock); Mockito.when(brokerControllerMock.getEndTransactionProcessor()).thenReturn(endTransactionProcessorMock); + Mockito.when(brokerControllerMock.getRecallMessageProcessor()).thenReturn(recallMessageProcessorMock); Mockito.when(brokerControllerMock.getBrokerConfig()).thenReturn(new BrokerConfig()); localMessageService = new LocalMessageService(brokerControllerMock, channelManager, null); proxyContext = ProxyContext.create().withVal(ContextVariable.REMOTE_ADDRESS, "0.0.0.1") @@ -424,6 +432,31 @@ public void testAckMessage() throws Exception { assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK); } + @Test + public void testRecallMessage_success() throws Exception { + RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader(); + responseHeader.setMsgId("msgId"); + RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SUCCESS, responseHeader); + Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), + Mockito.any())).thenReturn(response); + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + String msgId = localMessageService.recallMessage(proxyContext, "brokerName", requestHeader, 1000L).join(); + assertThat(msgId).isEqualTo("msgId"); + } + + @Test + public void testRecallMessage_fail() throws Exception { + RecallMessageResponseHeader responseHeader = new RecallMessageResponseHeader(); + RemotingCommand response = RemotingCommand.createResponseCommandWithHeader(ResponseCode.SLAVE_NOT_AVAILABLE, responseHeader); + Mockito.when(recallMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), + Mockito.any())).thenReturn(response); + RecallMessageRequestHeader requestHeader = new RecallMessageRequestHeader(); + CompletionException exception = Assert.assertThrows(CompletionException.class, () -> { + localMessageService.recallMessage(proxyContext, "brokerName", requestHeader, 1000L).join(); + }); + Assert.assertTrue(exception.getCause() instanceof ProxyException); + } + private MessageExt buildMessageExt(String topic, int queueId, long queueOffset) { MessageExt message1 = new MessageExt(); message1.setTopic(topic); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java index cfc5cc22785..9e86422c482 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RequestCode.java @@ -220,6 +220,7 @@ public class RequestCode { public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354; public static final int LITE_PULL_MESSAGE = 361; + public static final int RECALL_MESSAGE = 370; public static final int QUERY_ASSIGNMENT = 400; public static final int SET_MESSAGE_REQUEST_MODE = 401; diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java new file mode 100644 index 00000000000..c29883682a0 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageRequestHeader.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.remoting.protocol.header; + +import com.google.common.base.MoreObjects; +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.action.RocketMQAction; +import org.apache.rocketmq.common.resource.ResourceType; +import org.apache.rocketmq.common.resource.RocketMQResource; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.rpc.TopicRequestHeader; + +@RocketMQAction(value = RequestCode.RECALL_MESSAGE, action = Action.PUB) +public class RecallMessageRequestHeader extends TopicRequestHeader { + @CFNullable + private String producerGroup; + + @CFNotNull + @RocketMQResource(ResourceType.TOPIC) + private String topic; + + @CFNotNull + private String recallHandle; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getProducerGroup() { + return producerGroup; + } + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getRecallHandle() { + return recallHandle; + } + + public void setRecallHandle(String recallHandle) { + this.recallHandle = recallHandle; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("producerGroup", producerGroup) + .add("topic", topic) + .add("recallHandle", recallHandle) + .toString(); + } +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java new file mode 100644 index 00000000000..1833cfcd053 --- /dev/null +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/RecallMessageResponseHeader.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class RecallMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private String msgId; + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } +} diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java index fe1e8533e54..7563b910331 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/SendMessageResponseHeader.java @@ -36,6 +36,7 @@ public class SendMessageResponseHeader implements CommandCustomHeader, FastCodes private Long queueOffset; private String transactionId; private String batchUniqId; + private String recallHandle; @Override public void checkFields() throws RemotingCommandException { @@ -48,6 +49,7 @@ public void encode(ByteBuf out) { writeIfNotNull(out, "queueOffset", queueOffset); writeIfNotNull(out, "transactionId", transactionId); writeIfNotNull(out, "batchUniqId", batchUniqId); + writeIfNotNull(out, "recallHandle", recallHandle); } @Override @@ -76,6 +78,11 @@ public void decode(HashMap fields) throws RemotingCommandExcepti if (str != null) { this.batchUniqId = str; } + + str = fields.get("recallHandle"); + if (str != null) { + this.recallHandle = str; + } } public String getMsgId() { @@ -117,4 +124,12 @@ public String getBatchUniqId() { public void setBatchUniqId(String batchUniqId) { this.batchUniqId = batchUniqId; } + + public String getRecallHandle() { + return recallHandle; + } + + public void setRecallHandle(String recallHandle) { + this.recallHandle = recallHandle; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java index 2b14618eede..4287ce78ab0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java @@ -1599,7 +1599,8 @@ public void run() { if (null == uniqueKey) { LOGGER.warn("No uniqueKey for msg:{}", msgExt); } - if (null != uniqueKey && tr.getDeleteList() != null && tr.getDeleteList().size() > 0 && tr.getDeleteList().contains(uniqueKey)) { + if (null != uniqueKey && tr.getDeleteList() != null && tr.getDeleteList().size() > 0 + && tr.getDeleteList().contains(buildDeleteKey(getRealTopic(msgExt), uniqueKey))) { //Normally, it cancels out with the +1 above addMetric(msgExt, -1); doRes = true; @@ -1909,4 +1910,9 @@ public void setFrequency(AtomicInteger frequency) { public TimerCheckpoint getTimerCheckpoint() { return timerCheckpoint; } + + // identify a message by topic + uk, like query operation + public static String buildDeleteKey(String realTopic, String uniqueKey) { + return realTopic + "+" + uniqueKey; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java index 52e58efde23..36853bb44fe 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java @@ -359,7 +359,7 @@ public void testDeleteTimerMessage() throws Exception { MessageExtBrokerInner delMsg = buildMessage(delayMs, topic, false); transformTimerMessage(timerMessageStore,delMsg); - MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, uniqKey); + MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, uniqKey)); delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); @@ -374,6 +374,49 @@ public void testDeleteTimerMessage() throws Exception { assertNull(getOneMessage(topic, 0, 4, 500)); } + @Test + public void testDeleteTimerMessage_ukCollision() throws Exception { + String topic = "TimerTest_testDeleteTimerMessage"; + String collisionTopic = "TimerTest_testDeleteTimerMessage_collision"; + + TimerMessageStore timerMessageStore = createTimerMessageStore(null); + timerMessageStore.load(); + timerMessageStore.start(true); + + long curr = System.currentTimeMillis() / precisionMs * precisionMs; + long delayMs = curr + 1000; + + MessageExtBrokerInner inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + String firstUniqKey = MessageClientIDSetter.getUniqID(inner); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus()); + + inner = buildMessage(delayMs, topic, false); + transformTimerMessage(timerMessageStore, inner); + String secondUniqKey = MessageClientIDSetter.getUniqID(inner); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(inner).getPutMessageStatus()); + + MessageExtBrokerInner delMsg = buildMessage(delayMs, "whatever", false); + transformTimerMessage(timerMessageStore, delMsg); + MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(topic, firstUniqKey)); + delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); + + delMsg = buildMessage(delayMs, "whatever", false); + transformTimerMessage(timerMessageStore, delMsg); + MessageAccessor.putProperty(delMsg, TimerMessageStore.TIMER_DELETE_UNIQUE_KEY, TimerMessageStore.buildDeleteKey(collisionTopic, secondUniqKey)); + delMsg.setPropertiesString(MessageDecoder.messageProperties2String(delMsg.getProperties())); + assertEquals(PutMessageStatus.PUT_OK, messageStore.putMessage(delMsg).getPutMessageStatus()); + + // The first one should have been deleted, the second one should not be deleted. + ByteBuffer msgBuff = getOneMessage(topic, 0, 0, 3000); + assertNotNull(msgBuff); + MessageExt msgExt = MessageDecoder.decode(msgBuff); + assertNotNull(msgExt); + assertNotEquals(firstUniqKey, MessageClientIDSetter.getUniqID(msgExt)); + assertEquals(secondUniqKey, MessageClientIDSetter.getUniqID(msgExt)); + } + @Test public void testPutDeleteTimerMessage() throws Exception { String topic = "TimerTest_testPutDeleteTimerMessage"; diff --git a/test/BUILD.bazel b/test/BUILD.bazel index e6703d69a01..80bd06539e8 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -117,6 +117,8 @@ GenTestRules( "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendIT", "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithMQIT", "src/test/java/org/apache/rocketmq/test/offset/OffsetNotFoundIT", + "src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT", + "src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT", "src/test/java/org/apache/rocketmq/test/client/producer/querymsg/QueryMsgByIdIT", "src/test/java/org/apache/rocketmq/test/client/producer/oneway/OneWaySendWithSelectorIT", "src/test/java/org/apache/rocketmq/test/smoke/NormalMessageSendAndRecvIT", diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java index 77f5f362125..b754466a916 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java @@ -93,6 +93,11 @@ public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } + @Test + public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception { + super.testSimpleConsumerSendAndRecallDelayMessage(); + } + @Test public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { super.testSimpleConsumerSendAndRecvBigMessage(); diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java index 9d8f85b9981..534108c2805 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java @@ -41,6 +41,8 @@ import apache.rocketmq.v2.QueryAssignmentResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.RecallMessageRequest; +import apache.rocketmq.v2.RecallMessageResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.RecoverOrphanedTransactionCommand; @@ -393,6 +395,69 @@ public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { assertThat(Math.abs(recvTime.get() - sendTime - delayTime) < 2 * 1000).isTrue(); } + public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception { + String topic = initTopicOnSampleTopicBroker(BROKER1_NAME, TopicMessageType.DELAY); + String group = MQRandomUtils.getRandomConsumerGroup(); + long delayTime = TimeUnit.SECONDS.toMillis(5); + + // init consumer offset + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.DELAY) + .setBodyEncoding(Encoding.GZIP) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(NetworkUtil.getLocalAddress(), "127.0.0.1:1234")) + .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis() + delayTime)) + .build()) + .setBody(ByteString.copyFromUtf8("hello")) + .build()) + .build()); + long sendTime = System.currentTimeMillis(); + assertSendMessage(sendResponse, messageId); + String recallHandle = sendResponse.getEntries(0).getRecallHandle(); + assertThat(recallHandle).isNotEmpty(); + + RecallMessageRequest recallRequest = RecallMessageRequest.newBuilder() + .setRecallHandle(recallHandle) + .setTopic(Resource.newBuilder().setResourceNamespace("").setName(topic).build()) + .build(); + RecallMessageResponse recallResponse = + blockingStub.withDeadlineAfter(2, TimeUnit.SECONDS).recallMessage(recallRequest); + assertThat(recallResponse.getStatus()).isEqualTo( + ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); + assertThat(recallResponse.getMessageId()).isEqualTo(messageId); + + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + + AtomicLong recvTime = new AtomicLong(); + AtomicReference recvMessage = new AtomicReference<>(); + try { + await().atMost(java.time.Duration.ofSeconds(10)).until(() -> { + List messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group)); + if (messageList.isEmpty()) { + return false; + } + recvTime.set(System.currentTimeMillis()); + recvMessage.set(messageList.get(0)); + return messageList.get(0).getSystemProperties().getMessageId().equals(messageId); + }); + } catch (Exception e) { + } + assertThat(recvTime.get()).isEqualTo(0L); + assertThat(recvMessage.get()).isNull(); + } + public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { String topic = initTopicOnSampleTopicBroker(BROKER1_NAME); String group = MQRandomUtils.getRandomConsumerGroup(); @@ -427,6 +492,7 @@ public void testSimpleConsumerSendAndRecv() throws Exception { String messageId = createUniqID(); SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId)); assertSendMessage(sendResponse, messageId); + assertThat(sendResponse.getEntries(0).getRecallHandle()).isNullOrEmpty(); this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java index 515c3f121dd..5dd06f53420 100644 --- a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java @@ -81,6 +81,11 @@ public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { super.testSimpleConsumerSendAndRecvDelayMessage(); } + @Test + public void testSimpleConsumerSendAndRecallDelayMessage() throws Exception { + super.testSimpleConsumerSendAndRecallDelayMessage(); + } + @Test public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { super.testSimpleConsumerSendAndRecvBigMessage(); diff --git a/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java b/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java new file mode 100644 index 00000000000..d52c7002548 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/recall/RecallWithTraceIT.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.recall; + +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.trace.TraceContext; +import org.apache.rocketmq.client.trace.TraceDataEncoder; +import org.apache.rocketmq.client.trace.TraceType; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.producer.RecallMessageHandle; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.base.IntegrationTestBase; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; +import org.apache.rocketmq.test.factory.ConsumerFactory; +import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.awaitility.Awaitility.await; + +public class RecallWithTraceIT extends BaseConf { + private static String topic; + private static String group; + private static DefaultMQProducer producer; + private static RMQPopConsumer popConsumer; + + @BeforeClass + public static void init() throws MQClientException { + System.setProperty("com.rocketmq.recall.default.trace.enable", Boolean.TRUE.toString()); + topic = MQRandomUtils.getRandomTopic(); + IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.NORMAL); + group = initConsumerGroup(); + producer = new DefaultMQProducer(group, true, topic); + producer.setNamesrvAddr(NAMESRV_ADDR); + producer.start(); + popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, group, topic, "*", new RMQNormalListener()); + mqClients.add(popConsumer); + mqClients.add(producer); + } + + @AfterClass + public static void tearDown() { + shutdown(); + } + + @Test + public void testRecallTrace() throws MQBrokerException, RemotingException, InterruptedException, MQClientException { + String msgId = MessageClientIDSetter.createUniqID(); + String recallHandle = RecallMessageHandle.HandleV1.buildHandle(topic, BROKER1_NAME, + String.valueOf(System.currentTimeMillis() + 30000), msgId); + producer.recallMessage(topic, recallHandle); + + MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0); + String brokerAddress = brokerController1.getBrokerAddr(); + AtomicReference traceMessage = new AtomicReference(); + await() + .pollInterval(1, TimeUnit.SECONDS) + .atMost(15, TimeUnit.SECONDS) + .until(() -> { + PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1); + boolean found = popResult.getPopStatus().equals(PopStatus.FOUND); + traceMessage.set(found ? popResult.getMsgFoundList().get(0) : null); + return found; + }); + + Assert.assertNotNull(traceMessage.get()); + TraceContext context = + TraceDataEncoder.decoderFromTraceDataString(new String(traceMessage.get().getBody())).get(0); + Assert.assertEquals(TraceType.Recall, context.getTraceType()); + Assert.assertEquals(group, context.getGroupName()); + Assert.assertTrue(context.isSuccess()); + Assert.assertEquals(msgId, context.getTraceBeans().get(0).getMsgId()); + } +} diff --git a/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java b/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java new file mode 100644 index 00000000000..2fb9e023712 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/recall/SendAndRecallDelayMessageIT.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.recall; + +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.producer.RecallMessageHandle; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.base.IntegrationTestBase; +import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; +import org.apache.rocketmq.test.factory.ConsumerFactory; +import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.awaitility.Awaitility.await; + +public class SendAndRecallDelayMessageIT extends BaseConf { + + private static String initTopic; + private static String consumerGroup; + private static RMQNormalProducer producer; + private static RMQPopConsumer popConsumer; + + @BeforeClass + public static void init() { + initTopic = initTopic(); + consumerGroup = initConsumerGroup(); + producer = getProducer(NAMESRV_ADDR, initTopic); + popConsumer = ConsumerFactory.getRMQPopConsumer(NAMESRV_ADDR, consumerGroup, initTopic, "*", new RMQNormalListener()); + mqClients.add(popConsumer); + } + + @AfterClass + public static void tearDown() { + shutdown(); + } + + @Test + public void testSendAndRecv() throws Exception { + int delaySecond = 1; + String topic = MQRandomUtils.getRandomTopic(); + IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY); + MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0); + String brokerAddress = brokerController1.getBrokerAddr(); + + List sendList = buildSendMessageList(topic, delaySecond); + List recvList = new ArrayList<>(); + + for (Message message : sendList) { + producer.getProducer().send(message); + } + + await() + .pollInterval(1, TimeUnit.SECONDS) + .atMost(delaySecond + 15, TimeUnit.SECONDS) + .until(() -> { + PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1); + processPopResult(recvList, popResult); + return recvList.size() == sendList.size(); + }); + } + + @Test + public void testSendAndRecall() throws Exception { + int delaySecond = 5; + String topic = MQRandomUtils.getRandomTopic(); + IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY); + MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0); + String brokerAddress = brokerController1.getBrokerAddr(); + + List sendList = buildSendMessageList(topic, delaySecond); + List recvList = new ArrayList<>(); + int recallCount = 0; + + for (Message message : sendList) { + SendResult sendResult = producer.getProducer().send(message); + if (sendResult.getRecallHandle() != null) { + String messageId = producer.getProducer().recallMessage(topic, sendResult.getRecallHandle()); + assertEquals(sendResult.getMsgId(), messageId); + recallCount += 1; + } + } + assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message + try { + await() + .pollInterval(1, TimeUnit.SECONDS) + .atMost(delaySecond + 15, TimeUnit.SECONDS) + .until(() -> { + PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1); + processPopResult(recvList, popResult); + return recvList.size() == sendList.size(); + }); + } catch (Exception e) { + } + assertEquals(sendList.size() - recallCount, recvList.size()); + } + + @Test + public void testSendAndRecall_ukCollision() throws Exception { + int delaySecond = 5; + String topic = MQRandomUtils.getRandomTopic(); + String collisionTopic = MQRandomUtils.getRandomTopic(); + IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY); + IntegrationTestBase.initTopic(collisionTopic, NAMESRV_ADDR, BROKER1_NAME, 1, CQType.SimpleCQ, TopicMessageType.DELAY); + MessageQueue messageQueue = new MessageQueue(topic, BROKER1_NAME, 0); + String brokerAddress = brokerController1.getBrokerAddr(); + + List sendList = buildSendMessageList(topic, delaySecond); + List recvList = new ArrayList<>(); + int recallCount = 0; + + for (Message message : sendList) { + SendResult sendResult = producer.getProducer().send(message); + if (sendResult.getRecallHandle() != null) { + RecallMessageHandle.HandleV1 handleEntity = + (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(sendResult.getRecallHandle()); + String collisionHandle = RecallMessageHandle.HandleV1.buildHandle(collisionTopic, + handleEntity.getBrokerName(), handleEntity.getTimestampStr(), handleEntity.getMessageId()); + String messageId = producer.getProducer().recallMessage(collisionTopic, collisionHandle); + assertEquals(sendResult.getMsgId(), messageId); + recallCount += 1; + } + } + assertEquals(sendList.size() - 2, recallCount); // one normal and one delay-level message + + try { + await() + .pollInterval(1, TimeUnit.SECONDS) + .atMost(delaySecond + 15, TimeUnit.SECONDS) + .until(() -> { + PopResult popResult = popConsumer.pop(brokerAddress, messageQueue, 60 * 1000, -1); + processPopResult(recvList, popResult); + return recvList.size() == sendList.size(); + }); + } catch (Exception e) { + } + assertEquals(sendList.size(), recvList.size()); + } + + private void processPopResult(List recvList, PopResult popResult) { + if (popResult.getPopStatus() == PopStatus.FOUND && popResult.getMsgFoundList() != null) { + recvList.addAll(popResult.getMsgFoundList()); + } + } + + private List buildSendMessageList(String topic, int delaySecond) { + Message msg0 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); // not supported + + Message msg1 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); // not supported + msg1.setDelayTimeLevel(2); + + Message msg2 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); + msg2.setDelayTimeMs(delaySecond * 1000L); + + Message msg3 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); + msg3.setDelayTimeSec(delaySecond); + + Message msg4 = new Message(topic, "tag", "Hello RocketMQ".getBytes()); + msg4.setDeliverTimeMs(System.currentTimeMillis() + delaySecond * 1000L); + + return Arrays.asList(msg0, msg1, msg2, msg3, msg4); + } +} From 7da9ad4fa59da83f641a39bc8cd36233e3c7bb2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E9=99=88?= Date: Tue, 10 Dec 2024 14:40:27 +0800 Subject: [PATCH 258/265] [ISSUE #9042] Update createTimerMessageStore call with new parameter (#9041) * Update createTimerMessageStore call with new parameter * Reduce unit test execution time --- .../apache/rocketmq/store/timer/TimerMessageStoreTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java index 36853bb44fe..a014e77b90e 100644 --- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMessageStoreTest.java @@ -259,8 +259,7 @@ public void testRetryUntilSuccess() throws Exception { latch.countDown(); } }).start(); - latch.await(10, TimeUnit.SECONDS); - + latch.await(5, TimeUnit.SECONDS); assertTrue(timerMessageStore.dequeuePutQueue.isEmpty()); verify(mockMessageStore, times(6)).putMessage(any(MessageExtBrokerInner.class)); } @@ -379,7 +378,7 @@ public void testDeleteTimerMessage_ukCollision() throws Exception { String topic = "TimerTest_testDeleteTimerMessage"; String collisionTopic = "TimerTest_testDeleteTimerMessage_collision"; - TimerMessageStore timerMessageStore = createTimerMessageStore(null); + TimerMessageStore timerMessageStore = createTimerMessageStore(null , false); timerMessageStore.load(); timerMessageStore.start(true); From 9834691b73bbdf9f9f6ddc524915444ee46f492e Mon Sep 17 00:00:00 2001 From: yx9o Date: Tue, 10 Dec 2024 14:41:07 +0800 Subject: [PATCH 259/265] [ISSUE #9021] Correct the error message of acl command (#9022) --- .../processor/AdminBrokerProcessor.java | 72 ++++++++++--------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 4c341dde920..b9b8b06d7ac 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -45,6 +45,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.acl.AccessValidator; +import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.auth.authentication.enums.UserType; import org.apache.rocketmq.auth.authentication.exception.AuthenticationException; @@ -771,26 +772,15 @@ private void deleteTopicInBroker(String topic) { this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic)); } - private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final CreateAccessConfigRequestHeader requestHeader = - (CreateAccessConfigRequestHeader) request.decodeCommandCustomHeader(CreateAccessConfigRequestHeader.class); - - PlainAccessConfig accessConfig = new PlainAccessConfig(); - accessConfig.setAccessKey(requestHeader.getAccessKey()); - accessConfig.setSecretKey(requestHeader.getSecretKey()); - accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress()); - accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm()); - accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm()); - accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ",")); - accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ",")); - accessConfig.setAdmin(requestHeader.isAdmin()); try { + ensureAclEnabled(); + final CreateAccessConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateAccessConfigRequestHeader.class); AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); - if (accessValidator.updateAccessConfig(accessConfig)) { + if (accessValidator.updateAccessConfig(createAccessConfig(requestHeader))) { response.setCode(ResponseCode.SUCCESS); response.setOpaque(request.getOpaque()); response.markResponseType(); @@ -813,15 +803,28 @@ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerC return null; } - private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + private PlainAccessConfig createAccessConfig(final CreateAccessConfigRequestHeader requestHeader) { + PlainAccessConfig accessConfig = new PlainAccessConfig(); + accessConfig.setAccessKey(requestHeader.getAccessKey()); + accessConfig.setSecretKey(requestHeader.getSecretKey()); + accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress()); + accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm()); + accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm()); + accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ",")); + accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ",")); + accessConfig.setAdmin(requestHeader.isAdmin()); + return accessConfig; + } + + private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final DeleteAccessConfigRequestHeader requestHeader = - (DeleteAccessConfigRequestHeader) request.decodeCommandCustomHeader(DeleteAccessConfigRequestHeader.class); LOGGER.info("DeleteAccessConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); try { + ensureAclEnabled(); + + final DeleteAccessConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteAccessConfigRequestHeader.class); String accessKey = requestHeader.getAccessKey(); AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); if (accessValidator.deleteAccessConfig(accessKey)) { @@ -848,15 +851,13 @@ private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ct return null; } - private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - + private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader = - (UpdateGlobalWhiteAddrsConfigRequestHeader) request.decodeCommandCustomHeader(UpdateGlobalWhiteAddrsConfigRequestHeader.class); - try { + ensureAclEnabled(); + + final UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateGlobalWhiteAddrsConfigRequestHeader.class); AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.split(requestHeader.getGlobalWhiteAddrs(), ","), requestHeader.getAclFileFullPath())) { @@ -883,18 +884,12 @@ private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandler } private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, RemotingCommand request) { - final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerAclConfigResponseHeader.class); - if (!brokerController.getBrokerConfig().isAclEnable()) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("The broker does not enable acl."); - return response; - } - - final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader(); - try { + ensureAclEnabled(); + + final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader(); AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class); responseHeader.setVersion(accessValidator.getAclConfigVersion()); @@ -907,9 +902,16 @@ private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, Rem return response; } catch (Exception e) { LOGGER.error("Failed to generate a proper getBrokerAclConfigVersion response", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); + return response; } + } - return null; + private void ensureAclEnabled() { + if (!brokerController.getBrokerConfig().isAclEnable()) { + throw new AclException("The broker does not enable acl."); + } } private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) { From 564e55ea58ba10e366d1136b5381f10e5a5c58e0 Mon Sep 17 00:00:00 2001 From: weihubeats Date: Tue, 10 Dec 2024 14:54:42 +0800 Subject: [PATCH 260/265] [ISSUE #8970] Remove redundant heartbeats (#8971) --- .../client/impl/factory/MQClientInstance.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 8cc910487c1..eba654c22d0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.impl.factory; +import com.alibaba.fastjson.JSON; import io.netty.channel.Channel; import java.util.Collections; import java.util.HashMap; @@ -35,7 +36,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import com.alibaba.fastjson.JSON; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.admin.MQAdminExtInner; @@ -66,6 +66,8 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueAssignment; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.HeartbeatV2Result; @@ -83,8 +85,6 @@ import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.route.QueueData; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import static org.apache.rocketmq.remoting.rpc.ClientMetadata.topicRouteData2EndpointsForStaticTopic; @@ -157,7 +157,9 @@ public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String cli ChannelEventListener channelEventListener; if (clientConfig.isEnableHeartbeatChannelEventListener()) { channelEventListener = new ChannelEventListener() { + private final ConcurrentMap> brokerAddrTable = MQClientInstance.this.brokerAddrTable; + @Override public void onChannelConnect(String remoteAddr, Channel channel) { } @@ -182,7 +184,7 @@ public void onChannelActive(String remoteAddr, Channel channel) { if (addr.equals(remoteAddr)) { long id = entry.getKey(); String brokerName = addressEntry.getKey(); - if (sendHeartbeatToBroker(id, brokerName, addr)) { + if (sendHeartbeatToBroker(id, brokerName, addr, false)) { rebalanceImmediately(); } break; @@ -591,6 +593,18 @@ private boolean isBrokerAddrExistInTopicRouteTable(final String addr) { } public boolean sendHeartbeatToBroker(long id, String brokerName, String addr) { + return sendHeartbeatToBroker(id, brokerName, addr, true); + } + + /** + * @param id + * @param brokerName + * @param addr + * @param strictLockMode When the connection is initially established, sending a heartbeat will simultaneously trigger the onChannelActive event to acquire the lock again, causing an exception. Therefore, + * the exception that occurs when sending the heartbeat during the initial onChannelActive event can be ignored. + * @return + */ + public boolean sendHeartbeatToBroker(long id, String brokerName, String addr, boolean strictLockMode) { if (this.lockHeartbeat.tryLock()) { final HeartbeatData heartbeatDataWithSub = this.prepareHeartbeatData(false); final boolean producerEmpty = heartbeatDataWithSub.getProducerDataSet().isEmpty(); @@ -615,7 +629,9 @@ public boolean sendHeartbeatToBroker(long id, String brokerName, String addr) { this.lockHeartbeat.unlock(); } } else { - log.warn("lock heartBeat, but failed. [{}]", this.clientId); + if (strictLockMode) { + log.warn("lock heartBeat, but failed. [{}]", this.clientId); + } } return false; } From 9aa081b8acfd01a40f50bd9c3face3c0d2c530b1 Mon Sep 17 00:00:00 2001 From: guyinyou <36399867+guyinyou@users.noreply.github.com> Date: Tue, 10 Dec 2024 20:05:44 +0800 Subject: [PATCH 261/265] [ISSUE #8988] Support dispatchBehindMilliseconds (#8989) * support dispatchBehindMilliseconds * Modify the initial value of currentReputTimestamp --------- Co-authored-by: guyinyou --- .../rocketmq/store/DefaultMessageStore.java | 25 ++++++++++++++++++- .../apache/rocketmq/store/MessageStore.java | 7 ++++++ .../plugin/AbstractPluginMessageStore.java | 5 ++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 6b8ea0ee8ad..9d3c46a438a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -1556,6 +1556,10 @@ public boolean checkInStoreByConsumeOffset(String topic, int queueId, long consu public long dispatchBehindBytes() { return this.reputMessageService.behind(); } + @Override + public long dispatchBehindMilliseconds() { + return this.reputMessageService.behindMs(); + } public long flushBehindBytes() { if (this.messageStoreConfig.isTransientStorePoolEnable()) { @@ -2793,6 +2797,7 @@ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { class ReputMessageService extends ServiceThread { protected volatile long reputFromOffset = 0; + protected volatile long currentReputTimestamp = System.currentTimeMillis(); public long getReputFromOffset() { return reputFromOffset; @@ -2802,6 +2807,10 @@ public void setReputFromOffset(long reputFromOffset) { this.reputFromOffset = reputFromOffset; } + public long getCurrentReputTimestamp() { + return currentReputTimestamp; + } + @Override public void shutdown() { for (int i = 0; i < 50 && this.isCommitLogAvailable(); i++) { @@ -2824,6 +2833,15 @@ public long behind() { return DefaultMessageStore.this.getConfirmOffset() - this.reputFromOffset; } + public long behindMs() { + long lastCommitLogFileTimeStamp = System.currentTimeMillis(); + MappedFile lastMappedFile = DefaultMessageStore.this.commitLog.getMappedFileQueue().getLastMappedFile(); + if (lastMappedFile != null) { + lastCommitLogFileTimeStamp = lastMappedFile.getStoreTimestamp(); + } + return Math.max(0, lastCommitLogFileTimeStamp - this.currentReputTimestamp); + } + public boolean isCommitLogAvailable() { return this.reputFromOffset < getReputEndOffset(); } @@ -2838,7 +2856,11 @@ public void doReput() { this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); } - for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { + boolean isCommitLogAvailable = isCommitLogAvailable(); + if (!isCommitLogAvailable) { + currentReputTimestamp = System.currentTimeMillis(); + } + for (boolean doNext = true; isCommitLogAvailable && doNext; ) { SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset); @@ -2861,6 +2883,7 @@ public void doReput() { if (dispatchRequest.isSuccess()) { if (size > 0) { + currentReputTimestamp = dispatchRequest.getStoreTimestamp(); DefaultMessageStore.this.doDispatch(dispatchRequest); if (!notifyMessageArriveInBatch) { diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 5c3984e5b2c..4bbee142a17 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -511,6 +511,13 @@ CompletableFuture queryMessageAsync(final String topic, fina */ long dispatchBehindBytes(); + /** + * Get number of the milliseconds that have been stored in commit log and not yet dispatched to consume queue. + * + * @return number of the milliseconds to dispatch. + */ + long dispatchBehindMilliseconds(); + /** * Flush the message store to persist all data. * diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 0f57a17d463..d5d6236458e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -293,6 +293,11 @@ public long dispatchBehindBytes() { return next.dispatchBehindBytes(); } + @Override + public long dispatchBehindMilliseconds() { + return next.dispatchBehindMilliseconds(); + } + @Override public long flush() { return next.flush(); From b5cf3ca90aba1765519ccd3244dc31e3852051ea Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Sat, 14 Dec 2024 10:08:54 +0800 Subject: [PATCH 262/265] [ISSUE #9028] Adjust some error code for SYSTEM_ERROR (#9027) * feat: change some SYSTEM_ERROR to more meaningful error code Change-Id: I4b6ffa5aa18325eeadc29941c5788244c2770423 * feat: change some SYSTEM_ERROR to more meaningful error code Change-Id: I0c6ff75c5a2f7adde73261da93608781260e09da * test: adjust test case for error code Change-Id: I302ff5ad204280b55c8427ba4e8563b042263aeb * test: adjust test case for error code Change-Id: I7fc958c865c53b2a66b7bd77b6fb69f1546d2826 --- .../broker/client/net/Broker2Client.java | 2 +- .../AbstractSendMessageProcessor.java | 4 +-- .../processor/AdminBrokerProcessor.java | 30 +++++++++---------- .../processor/ConsumerManageProcessor.java | 4 +-- .../processor/NotificationProcessor.java | 2 +- .../processor/PeekMessageProcessor.java | 2 +- .../processor/PollingInfoProcessor.java | 2 +- .../broker/processor/PopMessageProcessor.java | 4 +-- .../processor/PullMessageProcessor.java | 2 +- .../broker/client/net/Broker2ClientTest.java | 2 +- .../processor/AdminBrokerProcessorTest.java | 16 +++++----- .../processor/PeekMessageProcessorTest.java | 2 +- .../remoting/protocol/ResponseCode.java | 2 ++ 13 files changed, 38 insertions(+), 36 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index f43f79b1be2..f8984963f94 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -113,7 +113,7 @@ public RemotingCommand resetOffset(String topic, String group, long timeStamp, b TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); if (null == topicConfig) { log.error("[reset-offset] reset offset failed, no topic in this broker. topic={}", topic); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); response.setRemark("[reset-offset] reset offset failed, no topic in this broker. topic=" + topic); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index ba2d1b5f320..39befedaa22 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -467,7 +467,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx, TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(requestHeader.getTopic()); if (!result.isValid()) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(result.getRemark()); return response; } @@ -522,7 +522,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); LOGGER.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index b9b8b06d7ac..ffac714c1ba 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -425,7 +425,7 @@ private RemotingCommand getSubscriptionGroup(ChannelHandlerContext ctx, SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(requestHeader.getGroup()); if (groupConfig == null) { LOGGER.error("No group in this broker, client: {} group: {}", ctx.channel().remoteAddress(), requestHeader.getGroup()); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); response.setRemark("No group in this broker"); return response; } @@ -514,13 +514,13 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext try { TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); if (!result.isValid()) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(result.getRemark()); return response; } if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { if (TopicValidator.isSystemTopic(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The topic[" + topic + "] is conflict with system topic."); return response; } @@ -541,7 +541,7 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(); String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey); if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("MIXED message type is not supported."); return response; } @@ -604,13 +604,13 @@ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerCont String topic = topicConfig.getTopicName(); TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); if (!result.isValid()) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(result.getRemark()); return response; } if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { if (TopicValidator.isSystemTopic(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The topic[" + topic + "] is conflict with system topic."); return response; } @@ -620,7 +620,7 @@ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerCont String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(); String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey); if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("MIXED message type is not supported."); return response; } @@ -674,13 +674,13 @@ private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerCo TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic); if (!result.isValid()) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(result.getRemark()); return response; } if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { if (TopicValidator.isSystemTopic(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The topic[" + topic + "] is conflict with system topic."); return response; } @@ -721,14 +721,14 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, String topic = requestHeader.getTopic(); if (UtilAll.isBlank(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The specified topic is blank."); return response; } if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) { if (TopicValidator.isSystemTopic(topic)) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The topic[" + topic + "] is conflict with system topic."); return response; } @@ -1092,7 +1092,7 @@ private RemotingCommand setCommitLogReadaheadMode(ChannelHandlerContext ctx, Rem } int mode = Integer.parseInt(extFields.get(FIleReadaheadMode.READ_AHEAD_MODE)); if (mode != LibC.MADV_RANDOM && mode != LibC.MADV_NORMAL) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("set commitlog readahead mode param value error"); return response; } @@ -3081,7 +3081,7 @@ private RemotingCommand createUser(ChannelHandlerContext ctx, CreateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateUserRequestHeader.class); if (StringUtils.isEmpty(requestHeader.getUsername())) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The username is blank"); return response; } @@ -3113,7 +3113,7 @@ private RemotingCommand updateUser(ChannelHandlerContext ctx, UpdateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateUserRequestHeader.class); if (StringUtils.isEmpty(requestHeader.getUsername())) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The username is blank"); return response; } @@ -3177,7 +3177,7 @@ private RemotingCommand getUser(ChannelHandlerContext ctx, GetUserRequestHeader requestHeader = request.decodeCommandCustomHeader(GetUserRequestHeader.class); if (StringUtils.isBlank(requestHeader.getUsername())) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("The username is blank"); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java index 9b3ef603de7..dfa755d7c44 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java @@ -177,13 +177,13 @@ private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, Remoting } if (queueId == null) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("QueueId is null, topic is " + topic); return response; } if (offset == null) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark("Offset is null, topic is " + topic); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java index b4ebd9c4a99..6317d6ad7d2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java @@ -112,7 +112,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); POP_LOGGER.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java index 8473e3a2865..40117b74a54 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java @@ -114,7 +114,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); LOG.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java index 65a4d7d7851..f7baac144e6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java @@ -89,7 +89,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); POP_LOGGER.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index e0454afa3ca..05efc14b7b4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -252,7 +252,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC return response; } if (requestHeader.getMaxMsgNums() > 32) { - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(String.format("the broker[%s] pop message's num is greater than 32", this.brokerController.getBrokerConfig().getBrokerIP1())); return response; @@ -288,7 +288,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); POP_LOGGER.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index 2ad2c9e93e4..5b11bc2fef4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -371,7 +371,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); LOGGER.warn(errorInfo); - response.setCode(ResponseCode.SYSTEM_ERROR); + response.setCode(ResponseCode.INVALID_PARAMETER); response.setRemark(errorInfo); return response; } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java index 7e16d329e1b..ccb489aead2 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java @@ -129,7 +129,7 @@ public void testCheckProducerTransactionStateException() throws Exception { public void testResetOffsetNoTopicConfig() throws RemotingCommandException { when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null); RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); - assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); + assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode()); } @Test diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index 48ddb891728..959b147d9d3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -317,7 +317,7 @@ public void testUpdateAndCreateTopic() throws Exception { for (String topic : systemTopicSet) { RemotingCommand request = buildCreateTopicRequest(topic); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic."); } @@ -325,7 +325,7 @@ public void testUpdateAndCreateTopic() throws Exception { String topic = ""; RemotingCommand request = buildCreateTopicRequest(topic); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); topic = "TEST_CREATE_TOPIC"; request = buildCreateTopicRequest(topic); @@ -339,7 +339,7 @@ public void testUpdateAndCreateTopic() throws Exception { attributes.put("+message.type", "MIXED"); request = buildCreateTopicRequest(topic, attributes); response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); // test allow MIXED topic type brokerController.getBrokerConfig().setEnableMixedMessageType(true); response = adminBrokerProcessor.processRequest(handlerContext, request); @@ -351,14 +351,14 @@ public void testUpdateAndCreateTopicList() throws RemotingCommandException { List systemTopicList = new ArrayList<>(systemTopicSet); RemotingCommand request = buildCreateTopicListRequest(systemTopicList); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); assertThat(response.getRemark()).isEqualTo("The topic[" + systemTopicList.get(0) + "] is conflict with system topic."); List inValidTopicList = new ArrayList<>(); inValidTopicList.add(""); request = buildCreateTopicListRequest(inValidTopicList); response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); List topicList = new ArrayList<>(); topicList.add("TEST_CREATE_TOPIC"); @@ -378,7 +378,7 @@ public void testUpdateAndCreateTopicList() throws RemotingCommandException { attributes.put("+message.type", "MIXED"); request = buildCreateTopicListRequest(topicList, attributes); response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); // test allow MIXED topic type brokerController.getBrokerConfig().setEnableMixedMessageType(true); response = adminBrokerProcessor.processRequest(handlerContext, request); @@ -400,7 +400,7 @@ public void testDeleteTopic() throws Exception { for (String topic : systemTopicSet) { RemotingCommand request = buildDeleteTopicRequest(topic); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); assertThat(response.getRemark()).isEqualTo("The topic[" + topic + "] is conflict with system topic."); } @@ -1065,7 +1065,7 @@ public void testSetCommitLogReadAheadMode() throws RemotingCommandException { extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_DONTNEED)); request.setExtFields(extfields); response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); extfields.clear(); extfields.put(FIleReadaheadMode.READ_AHEAD_MODE, String.valueOf(LibC.MADV_NORMAL)); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java index 7f8504453ca..9baf2a6ebb3 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PeekMessageProcessorTest.java @@ -154,7 +154,7 @@ public void testProcessRequest_SubscriptionGroupNotExist() throws RemotingComman public void testProcessRequest_QueueIdError() throws RemotingCommandException { RemotingCommand request = createPeekMessageRequest("group","topic",17); RemotingCommand response = peekMessageProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.INVALID_PARAMETER); } private RemotingCommand createPeekMessageRequest(String group,String topic,int queueId) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java index b19355487e5..e2ce81d95b9 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java @@ -55,6 +55,8 @@ public class ResponseCode extends RemotingSysResponseCode { public static final int FILTER_DATA_NOT_LATEST = 28; + public static final int INVALID_PARAMETER = 29; + public static final int TRANSACTION_SHOULD_COMMIT = 200; public static final int TRANSACTION_SHOULD_ROLLBACK = 201; From 93e268909cb92dd6d4033a4a36e1daab40025a75 Mon Sep 17 00:00:00 2001 From: mxsm Date: Sun, 15 Dec 2024 10:34:08 +0800 Subject: [PATCH 263/265] [ISSUE #9054] Optimize log print when client consume message in pop mode (#9055) --- .../client/impl/consumer/DefaultMQPushConsumerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 46715cea950..c93cff42452 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -510,7 +510,7 @@ void popMessage(final PopRequest popRequest) { try { this.makeSureStateOK(); } catch (MQClientException e) { - log.warn("pullMessage exception, consumer state not ok", e); + log.warn("popMessage exception, consumer state not ok", e); this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException); return; } From 16b6e53263477794125a49d9f31a994a510970b7 Mon Sep 17 00:00:00 2001 From: Lei Zhiyuan Date: Thu, 19 Dec 2024 11:44:32 +0800 Subject: [PATCH 264/265] [ISSUE #9002] when bytebuffer is not enough, do not throw exception (#9003) * fix: when bytebuffer is not enough,we should wait for next instead of throw exception * fix: when bytebuffer is not enough,we should wait for next instead of throw exception --- .../src/main/java/org/apache/rocketmq/store/CommitLog.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index d30691908b2..ff96bf1066b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -432,8 +432,14 @@ private void doNothingForDeadCode(final Object obj) { public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC, final boolean checkDupInfo, final boolean readBody) { try { + if (byteBuffer.remaining() <= 4) { + return new DispatchRequest(-1, false /* fail */); + } // 1 TOTAL SIZE int totalSize = byteBuffer.getInt(); + if (byteBuffer.remaining() < totalSize - 4) { + return new DispatchRequest(-1, false /* fail */); + } // 2 MAGIC CODE int magicCode = byteBuffer.getInt(); @@ -628,6 +634,7 @@ public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, return dispatchRequest; } catch (Exception e) { + log.error("checkMessageAndReturnSize failed, may can not dispatch", e); } return new DispatchRequest(-1, false /* success */); From 91fdc35db305e4db86894a195b86c6102853ae72 Mon Sep 17 00:00:00 2001 From: dingshuangxi888 Date: Fri, 20 Dec 2024 15:39:46 +0800 Subject: [PATCH 265/265] Fix the permission check for retry topic to get topic route. (#9073) --- .../builder/DefaultAuthorizationContextBuilder.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java index bf86892ea61..fababc0ee71 100644 --- a/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java +++ b/auth/src/main/java/org/apache/rocketmq/auth/authorization/builder/DefaultAuthorizationContextBuilder.java @@ -182,8 +182,13 @@ public List build(ChannelHandlerContext context, Re Resource group; switch (command.getCode()) { case RequestCode.GET_ROUTEINFO_BY_TOPIC: - topic = Resource.ofTopic(fields.get(TOPIC)); - result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.PUB, Action.SUB, Action.GET), sourceIp)); + if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) { + group = Resource.ofGroup(fields.get(TOPIC)); + result.add(DefaultAuthorizationContext.of(subject, group, Arrays.asList(Action.SUB, Action.GET), sourceIp)); + } else { + topic = Resource.ofTopic(fields.get(TOPIC)); + result.add(DefaultAuthorizationContext.of(subject, topic, Arrays.asList(Action.PUB, Action.SUB, Action.GET), sourceIp)); + } break; case RequestCode.SEND_MESSAGE: if (NamespaceUtil.isRetryTopic(fields.get(TOPIC))) {