cfDescriptors = new ArrayList<>();
+
+ ColumnFamilyOptions defaultOptions = ConfigHelper.createConfigColumnFamilyOptions();
+ this.cfOptions.add(defaultOptions);
+ cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, defaultOptions));
+
+ // Start RocksDB instance
+ open(cfDescriptors);
+
+ this.defaultCFHandle = cfHandles.get(0);
+ } catch (Exception e) {
+ AbstractRocksDBStorage.LOGGER.error("postLoad Failed. {}", this.dbPath, e);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ protected void preShutdown() {
+ scheduledExecutorService.shutdown();
+ flushSyncService.shutdown();
+ }
+
+ protected void initOptions() {
+ this.options = ConfigHelper.createConfigDBOptions();
+ super.initOptions();
+ }
+
+ @Override
+ protected void initAbleWalWriteOptions() {
+ this.ableWalWriteOptions = new WriteOptions();
+
+ // Given that fdatasync is kind of expensive, sync-WAL for every write cannot be afforded.
+ this.ableWalWriteOptions.setSync(false);
+
+ // We need WAL for config changes
+ this.ableWalWriteOptions.setDisableWAL(false);
+
+ // No fast failure on block, wait synchronously even if there is wait for the write request
+ this.ableWalWriteOptions.setNoSlowdown(false);
+ }
+
+ public byte[] get(ByteBuffer key) throws RocksDBException {
+ byte[] keyBytes = new byte[key.remaining()];
+ key.get(keyBytes);
+ return super.get(getDefaultCFHandle(), totalOrderReadOptions, keyBytes);
+ }
+
+ public void write(WriteBatch writeBatch) throws RocksDBException {
+ db.write(ableWalWriteOptions, writeBatch);
+ accountWriteOps(writeBatch.getDataSize());
+ }
+
+ private void accountWriteOps(long dataSize) {
+ writeOpsCounter.incrementAndGet();
+ estimateWalFileSize.addAndGet(dataSize);
+ }
+
+ public RocksIterator iterate(ByteBuffer beginKey, ByteBuffer endKey) {
+ try (ReadOptions readOptions = new ReadOptions()) {
+ readOptions.setTotalOrderSeek(true);
+ readOptions.setTailing(false);
+ readOptions.setAutoPrefixMode(true);
+ // Use DirectSlice till the follow issue is fixed:
+ // https://github.com/facebook/rocksdb/issues/13098
+ //
+ // readOptions.setIterateUpperBound(new DirectSlice(endKey));
+ byte[] buf = new byte[endKey.remaining()];
+ endKey.slice().get(buf);
+ readOptions.setIterateUpperBound(new Slice(buf));
+
+ RocksIterator iterator = db.newIterator(defaultCFHandle, readOptions);
+ iterator.seek(beginKey.slice());
+ return iterator;
+ }
+ }
+
+ /**
+ * RocksDB writes contain 3 stages: application memory buffer --> OS Page Cache --> Disk.
+ * Given that we are having DBOptions::manual_wal_flush, we need to manually call DB::FlushWAL and DB::SyncWAL
+ * Note: DB::FlushWAL(true) will internally call DB::SyncWAL.
+ *
+ * See Flush And Sync WAL
+ */
+ class FlushSyncService extends ServiceThread {
+
+ private long lastSyncTime = 0;
+
+ private static final long MAX_SYNC_INTERVAL_IN_MILLIS = 100;
+
+ private final Stopwatch stopwatch = Stopwatch.createUnstarted();
+
+ private final FlushOptions flushOptions = new FlushOptions();
+
+ @Override
+ public String getServiceName() {
+ return "FlushSyncService";
+ }
+
+ @Override
+ public void run() {
+ flushOptions.setAllowWriteStall(false);
+ flushOptions.setWaitForFlush(true);
+ log.info("{} service started", this.getServiceName());
+ while (!this.isStopped()) {
+ try {
+ this.waitForRunning(10);
+ this.flushAndSyncWAL(false);
+ } catch (Exception e) {
+ log.warn("{} service has exception. ", this.getServiceName(), e);
+ }
+ }
+ try {
+ flushAndSyncWAL(true);
+ } catch (Exception e) {
+ log.warn("{} raised an exception while performing flush-and-sync WAL on exit",
+ this.getServiceName(), e);
+ }
+ flushOptions.close();
+ log.info("{} service end", this.getServiceName());
+ }
+
+ private void flushAndSyncWAL(boolean onExit) throws RocksDBException {
+ int writeOps = writeOpsCounter.get();
+ if (0 == writeOps) {
+ // No write ops to flush
+ return;
+ }
+
+ /*
+ * Normally, when MemTables become full then immutable, RocksDB threads will automatically flush them to L0
+ * SST files. The use case here is different: the MemTable may never get full and immutable given that the
+ * volume of data involved is relatively small. Further, we are constantly modifying the key-value pairs and
+ * generating WAL entries. The WAL file size can grow up to dozens of gigabytes without manual triggering of
+ * flush.
+ */
+ if (ConfigStorage.this.estimateWalFileSize.get() >= messageStoreConfig.getRocksdbWalFileRollingThreshold()) {
+ ConfigStorage.this.flush(flushOptions);
+ estimateWalFileSize.set(0L);
+ }
+
+ // Flush and Sync WAL if we have committed enough writes
+ if (writeOps >= messageStoreConfig.getRocksdbFlushWalFrequency() || onExit) {
+ stopwatch.reset().start();
+ ConfigStorage.this.db.flushWal(true);
+ long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+ writeOpsCounter.getAndAdd(-writeOps);
+ lastSyncTime = System.currentTimeMillis();
+ LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps);
+ return;
+ }
+ // Flush and Sync WAL if some writes are out there for a period of time
+ long elapsedTime = System.currentTimeMillis() - lastSyncTime;
+ if (elapsedTime > MAX_SYNC_INTERVAL_IN_MILLIS) {
+ stopwatch.reset().start();
+ ConfigStorage.this.db.flushWal(true);
+ long elapsed = stopwatch.stop().elapsed(TimeUnit.MILLISECONDS);
+ LOGGER.debug("Flush and Sync WAL of RocksDB[{}] costs {}ms, write-ops={}", dbPath, elapsed, writeOps);
+ writeOpsCounter.getAndAdd(-writeOps);
+ lastSyncTime = System.currentTimeMillis();
+ }
+ }
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java
new file mode 100644
index 00000000000..1821c801cbc
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/ConsumerOffsetManagerV2.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.util.internal.PlatformDependent;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.store.MessageStore;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+/**
+ *
+ * Layout of consumer offset key:
+ * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+ * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]
+ *
+ *
+ *
+ * Layout of consumer offset value: [offset, 8 bytes]
+ *
+ */
+public class ConsumerOffsetManagerV2 extends ConsumerOffsetManager {
+
+ private final ConfigStorage configStorage;
+
+ public ConsumerOffsetManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+ super(brokerController);
+ this.configStorage = configStorage;
+ }
+
+ @Override
+ protected void removeConsumerOffset(String topicAtGroup) {
+ if (!MixAll.isLmq(topicAtGroup)) {
+ super.removeConsumerOffset(topicAtGroup);
+ }
+
+ String[] topicGroup = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+ if (topicGroup.length != 2) {
+ LOG.error("Invalid topic group: {}", topicAtGroup);
+ return;
+ }
+
+ byte[] topicBytes = topicGroup[0].getBytes(StandardCharsets.UTF_8);
+ byte[] groupBytes = topicGroup[1].getBytes(StandardCharsets.UTF_8);
+
+ int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+ + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */
+ + Short.BYTES + topicBytes.length + 1;
+ // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group-bytes][CTRL_1, 1 byte]
+ // [topic-len, 2 bytes][topic-bytes][CTRL_1]
+ ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ beginKey.writeByte(TablePrefix.TABLE.getValue());
+ beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ beginKey.writeByte(RecordPrefix.DATA.getValue());
+ beginKey.writeShort(groupBytes.length);
+ beginKey.writeBytes(groupBytes);
+ beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+ beginKey.writeShort(topicBytes.length);
+ beginKey.writeBytes(topicBytes);
+ beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+
+ ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ endKey.writeByte(TablePrefix.TABLE.getValue());
+ endKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ endKey.writeByte(RecordPrefix.DATA.getValue());
+ endKey.writeShort(groupBytes.length);
+ endKey.writeBytes(groupBytes);
+ endKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+ endKey.writeShort(topicBytes.length);
+ endKey.writeBytes(topicBytes);
+ endKey.writeByte(AbstractRocksDBStorage.CTRL_2);
+
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here
+ writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey));
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ LOG.error("Failed to removeConsumerOffset, topicAtGroup={}", topicAtGroup, e);
+ } finally {
+ beginKey.release();
+ endKey.release();
+ }
+ }
+
+ @Override
+ public void removeOffset(String group) {
+ if (!MixAll.isLmq(group)) {
+ super.removeOffset(group);
+ }
+
+ byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+ int keyLen = 1 /* table-prefix */ + Short.BYTES /* table-id */ + 1 /* record-prefix */
+ + Short.BYTES /* group-len */ + groupBytes.length + 1 /* CTRL_1 */;
+
+ // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+ ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ beginKey.writeByte(TablePrefix.TABLE.getValue());
+ beginKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ beginKey.writeByte(RecordPrefix.DATA.getValue());
+ beginKey.writeShort(groupBytes.length);
+ beginKey.writeBytes(groupBytes);
+ beginKey.writeByte(AbstractRocksDBStorage.CTRL_1);
+
+ ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ endKey.writeByte(TablePrefix.TABLE.getValue());
+ endKey.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ endKey.writeByte(RecordPrefix.DATA.getValue());
+ endKey.writeShort(groupBytes.length);
+ endKey.writeBytes(groupBytes);
+ endKey.writeByte(AbstractRocksDBStorage.CTRL_2);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ // TODO: we have to make a copy here as WriteBatch lacks ByteBuffer API here
+ writeBatch.deleteRange(ConfigHelper.readBytes(beginKey), ConfigHelper.readBytes(endKey));
+ MessageStore messageStore = brokerController.getMessageStore();
+ long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ LOG.error("Failed to consumer offsets by group={}", group, e);
+ } finally {
+ beginKey.release();
+ endKey.release();
+ }
+ }
+
+ /**
+ *
+ * Layout of consumer offset key:
+ * [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte][group-len, 2 bytes][group bytes][CTRL_1, 1 byte]
+ * [topic-len, 2 bytes][topic bytes][CTRL_1, 1 byte][queue-id, 4 bytes]
+ *
+ *
+ *
+ * Layout of consumer offset value:
+ * [offset, 8 bytes]
+ *
+ *
+ * @param clientHost The client that submits consumer offsets
+ * @param group Group name
+ * @param topic Topic name
+ * @param queueId Queue ID
+ * @param offset Consumer offset of the specified queue
+ */
+ @Override
+ public void commitOffset(String clientHost, String group, String topic, int queueId, long offset) {
+ String key = topic + TOPIC_GROUP_SEPARATOR + group;
+
+ // We maintain a copy of classic consumer offset table in memory as they take very limited memory footprint.
+ // For LMQ offsets, given the volume and number of these type of records, they are maintained in RocksDB
+ // directly. Frequently used LMQ consumer offsets should reside either in block-cache or MemTable, so read/write
+ // should be blazingly fast.
+ if (!MixAll.isLmq(topic)) {
+ if (offsetTable.containsKey(key)) {
+ offsetTable.get(key).put(queueId, offset);
+ } else {
+ ConcurrentMap map = new ConcurrentHashMap<>();
+ ConcurrentMap prev = offsetTable.putIfAbsent(key, map);
+ if (null != prev) {
+ map = prev;
+ }
+ map.put(queueId, offset);
+ }
+ }
+
+ ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);
+ ByteBuf valueBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(Long.BYTES);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ valueBuf.writeLong(offset);
+ writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+ MessageStore messageStore = brokerController.getMessageStore();
+ long stateMachineVersion = messageStore != null ? messageStore.getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.CONSUMER_OFFSET, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ LOG.error("Failed to commit consumer offset", e);
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+ }
+
+ private ByteBuf keyOfConsumerOffset(String group, String topic, int queueId) {
+ byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+ byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
+ int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/
+ + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/
+ + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/
+ + Integer.BYTES /*queue-id*/;
+ ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ keyBuf.writeByte(TablePrefix.TABLE.getValue());
+ keyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ keyBuf.writeByte(RecordPrefix.DATA.getValue());
+ keyBuf.writeShort(groupBytes.length);
+ keyBuf.writeBytes(groupBytes);
+ keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+ keyBuf.writeShort(topicBytes.length);
+ keyBuf.writeBytes(topicBytes);
+ keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+ keyBuf.writeInt(queueId);
+ return keyBuf;
+ }
+
+ private ByteBuf keyOfPullOffset(String group, String topic, int queueId) {
+ byte[] groupBytes = group.getBytes(StandardCharsets.UTF_8);
+ byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
+ int keyLen = 1 /*table prefix*/ + Short.BYTES /*table-id*/ + 1 /*record-prefix*/
+ + Short.BYTES /*group-len*/ + groupBytes.length + 1 /*CTRL_1*/
+ + 2 /*topic-len*/ + topicBytes.length + 1 /* CTRL_1*/
+ + Integer.BYTES /*queue-id*/;
+ ByteBuf keyBuf = ConfigStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ keyBuf.writeByte(TablePrefix.TABLE.getValue());
+ keyBuf.writeShort(TableId.PULL_OFFSET.getValue());
+ keyBuf.writeByte(RecordPrefix.DATA.getValue());
+ keyBuf.writeShort(groupBytes.length);
+ keyBuf.writeBytes(groupBytes);
+ keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+ keyBuf.writeShort(topicBytes.length);
+ keyBuf.writeBytes(topicBytes);
+ keyBuf.writeByte(AbstractRocksDBStorage.CTRL_1);
+ keyBuf.writeInt(queueId);
+ return keyBuf;
+ }
+
+ @Override
+ public boolean load() {
+ return loadDataVersion() && loadConsumerOffsets();
+ }
+
+ @Override
+ public synchronized void persist() {
+ try {
+ configStorage.flushWAL();
+ } catch (RocksDBException e) {
+ LOG.error("Failed to flush RocksDB config instance WAL", e);
+ }
+ }
+
+ /**
+ *
+ * Layout of data version key:
+ * [table-prefix, 1 byte][table-id, 2 byte][record-prefix, 1 byte][data-version-bytes]
+ *
+ *
+ *
+ * Layout of data version value:
+ * [state-machine-version, 8 bytes][timestamp, 8 bytes][sequence counter, 8 bytes]
+ *
+ */
+ public boolean loadDataVersion() {
+ try {
+ ConfigHelper.loadDataVersion(configStorage, TableId.CONSUMER_OFFSET)
+ .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));
+ } catch (RocksDBException e) {
+ LOG.error("Failed to load RocksDB config", e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean loadConsumerOffsets() {
+ // [table-prefix, 1 byte][table-id, 2 bytes][record-prefix, 1 byte]
+ ByteBuf beginKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);
+ beginKeyBuf.writeByte(TablePrefix.TABLE.getValue());
+ beginKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ beginKeyBuf.writeByte(RecordPrefix.DATA.getValue());
+
+ ByteBuf endKeyBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(4);
+ endKeyBuf.writeByte(TablePrefix.TABLE.getValue());
+ endKeyBuf.writeShort(TableId.CONSUMER_OFFSET.getValue());
+ endKeyBuf.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+ try (RocksIterator iterator = configStorage.iterate(beginKeyBuf.nioBuffer(), endKeyBuf.nioBuffer())) {
+ int keyCapacity = 256;
+ // We may iterate millions of LMQ consumer offsets here, use direct byte buffers here to avoid memory
+ // fragment
+ ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyCapacity);
+ ByteBuffer valueBuffer = ByteBuffer.allocateDirect(Long.BYTES);
+ while (iterator.isValid()) {
+ keyBuffer.clear();
+ valueBuffer.clear();
+
+ int len = iterator.key(keyBuffer);
+ if (len > keyCapacity) {
+ keyCapacity = len;
+ PlatformDependent.freeDirectBuffer(keyBuffer);
+ // Reserve more space for key
+ keyBuffer = ByteBuffer.allocateDirect(keyCapacity);
+ continue;
+ }
+ len = iterator.value(valueBuffer);
+ assert len == Long.BYTES;
+
+ // skip table-prefix, table-id, record-prefix
+ keyBuffer.position(1 + 2 + 1);
+ short groupLen = keyBuffer.getShort();
+ byte[] groupBytes = new byte[groupLen];
+ keyBuffer.get(groupBytes);
+ byte ctrl = keyBuffer.get();
+ assert ctrl == AbstractRocksDBStorage.CTRL_1;
+
+ short topicLen = keyBuffer.getShort();
+ byte[] topicBytes = new byte[topicLen];
+ keyBuffer.get(topicBytes);
+ String topic = new String(topicBytes, StandardCharsets.UTF_8);
+ ctrl = keyBuffer.get();
+ assert ctrl == AbstractRocksDBStorage.CTRL_1;
+
+ int queueId = keyBuffer.getInt();
+
+ long offset = valueBuffer.getLong();
+
+ if (!MixAll.isLmq(topic)) {
+ String group = new String(groupBytes, StandardCharsets.UTF_8);
+ onConsumerOffsetRecordLoad(topic, group, queueId, offset);
+ }
+ iterator.next();
+ }
+ PlatformDependent.freeDirectBuffer(keyBuffer);
+ PlatformDependent.freeDirectBuffer(valueBuffer);
+ } finally {
+ beginKeyBuf.release();
+ endKeyBuf.release();
+ }
+ return true;
+ }
+
+ private void onConsumerOffsetRecordLoad(String topic, String group, int queueId, long offset) {
+ if (MixAll.isLmq(topic)) {
+ return;
+ }
+ String key = topic + TOPIC_GROUP_SEPARATOR + group;
+ if (!offsetTable.containsKey(key)) {
+ ConcurrentMap map = new ConcurrentHashMap<>();
+ offsetTable.putIfAbsent(key, map);
+ }
+ offsetTable.get(key).put(queueId, offset);
+ }
+
+ @Override
+ public long queryOffset(String group, String topic, int queueId) {
+ if (!MixAll.isLmq(topic)) {
+ return super.queryOffset(group, topic, queueId);
+ }
+
+ ByteBuf keyBuf = keyOfConsumerOffset(group, topic, queueId);
+ try {
+ byte[] slice = configStorage.get(keyBuf.nioBuffer());
+ if (null == slice) {
+ return -1;
+ }
+ assert slice.length == Long.BYTES;
+ return ByteBuffer.wrap(slice).getLong();
+ } catch (RocksDBException e) {
+ throw new RuntimeException(e);
+ } finally {
+ keyBuf.release();
+ }
+ }
+
+ @Override
+ public void commitPullOffset(String clientHost, String group, String topic, int queueId, long offset) {
+ if (!MixAll.isLmq(topic)) {
+ super.commitPullOffset(clientHost, group, topic, queueId, offset);
+ }
+
+ ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);
+ ByteBuf valueBuf = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(8);
+ valueBuf.writeLong(offset);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.PULL_OFFSET, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ LOG.error("Failed to commit pull offset. group={}, topic={}, queueId={}, offset={}",
+ group, topic, queueId, offset);
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+ }
+
+ @Override
+ public long queryPullOffset(String group, String topic, int queueId) {
+ if (!MixAll.isLmq(topic)) {
+ return super.queryPullOffset(group, topic, queueId);
+ }
+
+ ByteBuf keyBuf = keyOfPullOffset(group, topic, queueId);
+ try {
+ byte[] valueBytes = configStorage.get(keyBuf.nioBuffer());
+ if (null == valueBytes) {
+ return -1;
+ }
+ return ByteBuffer.wrap(valueBytes).getLong();
+ } catch (RocksDBException e) {
+ LOG.error("Failed to queryPullOffset. group={}, topic={}, queueId={}", group, topic, queueId);
+ } finally {
+ keyBuf.release();
+ }
+ return -1;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java
new file mode 100644
index 00000000000..750d454d4ee
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/RecordPrefix.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+public enum RecordPrefix {
+ UNSPECIFIED((byte)0),
+ DATA_VERSION((byte)1),
+ DATA((byte)2);
+
+ private final byte value;
+
+ RecordPrefix(byte value) {
+ this.value = value;
+ }
+
+ public byte getValue() {
+ return value;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java
new file mode 100644
index 00000000000..2ee157fdc8d
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SerializationType.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+public enum SerializationType {
+ UNSPECIFIED((byte) 0),
+
+ JSON((byte) 1),
+
+ PROTOBUF((byte) 2),
+
+ FLAT_BUFFERS((byte) 3);
+
+ private final byte value;
+
+ SerializationType(byte value) {
+ this.value = value;
+ }
+
+ public byte getValue() {
+ return value;
+ }
+
+ public static SerializationType valueOf(byte value) {
+ for (SerializationType type : SerializationType.values()) {
+ if (type.getValue() == value) {
+ return type;
+ }
+ }
+ return SerializationType.UNSPECIFIED;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java
new file mode 100644
index 00000000000..dd67871f184
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/SubscriptionGroupManagerV2.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.alibaba.fastjson2.JSON;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+public class SubscriptionGroupManagerV2 extends SubscriptionGroupManager {
+
+ private final ConfigStorage configStorage;
+
+ public SubscriptionGroupManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+ super(brokerController);
+ this.configStorage = configStorage;
+ }
+
+ @Override
+ public boolean load() {
+ return loadDataVersion() && loadSubscriptions();
+ }
+
+ public boolean loadDataVersion() {
+ try {
+ ConfigHelper.loadDataVersion(configStorage, TableId.SUBSCRIPTION_GROUP)
+ .ifPresent(buf -> {
+ ConfigHelper.onDataVersionLoad(buf, dataVersion);
+ });
+ } catch (RocksDBException e) {
+ log.error("loadDataVersion error", e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean loadSubscriptions() {
+ int keyLen = 1 /* table prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;
+ ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ beginKey.writeByte(TablePrefix.TABLE.getValue());
+ beginKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());
+ beginKey.writeByte(RecordPrefix.DATA.getValue());
+
+ ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ endKey.writeByte(TablePrefix.TABLE.getValue());
+ endKey.writeShort(TableId.SUBSCRIPTION_GROUP.getValue());
+ endKey.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+ try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {
+ while (iterator.isValid()) {
+ SubscriptionGroupConfig subscriptionGroupConfig = parseSubscription(iterator.key(), iterator.value());
+ if (null != subscriptionGroupConfig) {
+ super.putSubscriptionGroupConfig(subscriptionGroupConfig);
+ }
+ iterator.next();
+ }
+ } finally {
+ beginKey.release();
+ endKey.release();
+ }
+ return true;
+ }
+
+ private SubscriptionGroupConfig parseSubscription(byte[] key, byte[] value) {
+ ByteBuf keyBuf = Unpooled.wrappedBuffer(key);
+ ByteBuf valueBuf = Unpooled.wrappedBuffer(value);
+ try {
+ // Skip table-prefix, table-id, record-type-prefix
+ keyBuf.readerIndex(4);
+ short groupNameLen = keyBuf.readShort();
+ assert groupNameLen == keyBuf.readableBytes();
+ CharSequence groupName = keyBuf.readCharSequence(groupNameLen, StandardCharsets.UTF_8);
+ assert null != groupName;
+ byte serializationType = valueBuf.readByte();
+ if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {
+ CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);
+ SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(json.toString(), SubscriptionGroupConfig.class);
+ assert subscriptionGroupConfig != null;
+ assert groupName.equals(subscriptionGroupConfig.getGroupName());
+ return subscriptionGroupConfig;
+ }
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+ return null;
+ }
+
+ @Override
+ public synchronized void persist() {
+ try {
+ configStorage.flushWAL();
+ } catch (RocksDBException e) {
+ log.error("Failed to flush RocksDB WAL", e);
+ }
+ }
+
+ @Override
+ public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {
+ if (MixAll.isLmq(group)) {
+ SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+ subscriptionGroupConfig.setGroupName(group);
+ return subscriptionGroupConfig;
+ }
+ return super.findSubscriptionGroupConfig(group);
+ }
+
+ @Override
+ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {
+ if (config == null || MixAll.isLmq(config.getGroupName())) {
+ return;
+ }
+ ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, config.getGroupName());
+ ByteBuf valueBuf = ConfigHelper.valueBufOf(config, SerializationType.JSON);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ // fdatasync on core metadata change
+ persist();
+ } catch (RocksDBException e) {
+ log.error("update subscription group config error", e);
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+ super.updateSubscriptionGroupConfigWithoutPersist(config);
+ }
+
+ @Override
+ public boolean containsSubscriptionGroup(String group) {
+ if (MixAll.isLmq(group)) {
+ return true;
+ } else {
+ return super.containsSubscriptionGroup(group);
+ }
+ }
+
+ @Override
+ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
+ ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.SUBSCRIPTION_GROUP, groupName);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ writeBatch.delete(ConfigHelper.readBytes(keyBuf));
+ long stateMachineVersion = brokerController.getMessageStore().getStateMachineVersion();
+ ConfigHelper.stampDataVersion(writeBatch, TableId.SUBSCRIPTION_GROUP, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ log.error("Failed to remove subscription group config by group-name={}", groupName, e);
+ }
+ return super.removeSubscriptionGroupConfig(groupName);
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java
new file mode 100644
index 00000000000..7a61899371e
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TableId.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+/**
+ * See Table, Key Value Mapping
+ */
+public enum TableId {
+ UNSPECIFIED((short) 0),
+ CONSUMER_OFFSET((short) 1),
+ PULL_OFFSET((short) 2),
+ TOPIC((short) 3),
+ SUBSCRIPTION_GROUP((short) 4);
+
+ private final short value;
+
+ TableId(short value) {
+ this.value = value;
+ }
+
+ public short getValue() {
+ return value;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java
new file mode 100644
index 00000000000..d16c14d275a
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TablePrefix.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+public enum TablePrefix {
+ UNSPECIFIED((byte) 0),
+ TABLE((byte) 1);
+
+ private final byte value;
+
+ TablePrefix(byte value) {
+ this.value = value;
+ }
+
+ public byte getValue() {
+ return value;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java
new file mode 100644
index 00000000000..7991d704459
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/TopicConfigManagerV2.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+import com.alibaba.fastjson2.JSON;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.nio.charset.StandardCharsets;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.config.AbstractRocksDBStorage;
+import org.apache.rocketmq.common.constant.PermName;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.WriteBatch;
+
+/**
+ * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]
+ * Value layout: [serialization-type, 1 byte][topic-config-bytes]
+ */
+public class TopicConfigManagerV2 extends TopicConfigManager {
+ private final ConfigStorage configStorage;
+
+ public TopicConfigManagerV2(BrokerController brokerController, ConfigStorage configStorage) {
+ super(brokerController);
+ this.configStorage = configStorage;
+ }
+
+ @Override
+ public boolean load() {
+ return loadDataVersion() && loadTopicConfig();
+ }
+
+ public boolean loadDataVersion() {
+ try {
+ ConfigHelper.loadDataVersion(configStorage, TableId.TOPIC)
+ .ifPresent(buf -> ConfigHelper.onDataVersionLoad(buf, dataVersion));
+ } catch (RocksDBException e) {
+ log.error("Failed to load data version of topic", e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean loadTopicConfig() {
+ int keyLen = 1 /* table-prefix */ + 2 /* table-id */ + 1 /* record-type-prefix */;
+ ByteBuf beginKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ beginKey.writeByte(TablePrefix.TABLE.getValue());
+ beginKey.writeShort(TableId.TOPIC.getValue());
+ beginKey.writeByte(RecordPrefix.DATA.getValue());
+
+ ByteBuf endKey = AbstractRocksDBStorage.POOLED_ALLOCATOR.buffer(keyLen);
+ endKey.writeByte(TablePrefix.TABLE.getValue());
+ endKey.writeShort(TableId.TOPIC.getValue());
+ endKey.writeByte(RecordPrefix.DATA.getValue() + 1);
+
+ try (RocksIterator iterator = configStorage.iterate(beginKey.nioBuffer(), endKey.nioBuffer())) {
+ while (iterator.isValid()) {
+ byte[] key = iterator.key();
+ byte[] value = iterator.value();
+ TopicConfig topicConfig = parseTopicConfig(key, value);
+ if (null != topicConfig) {
+ super.putTopicConfig(topicConfig);
+ }
+ iterator.next();
+ }
+ } finally {
+ beginKey.release();
+ endKey.release();
+ }
+ return true;
+ }
+
+ /**
+ * Key layout: [table-prefix, 1 byte][table-id, 2 bytes][record-type-prefix, 1 byte][topic-len, 2 bytes][topic-bytes]
+ * Value layout: [serialization-type, 1 byte][topic-config-bytes]
+ *
+ * @param key Topic config key representation in RocksDB
+ * @param value Topic config value representation in RocksDB
+ * @return decoded topic config
+ */
+ private TopicConfig parseTopicConfig(byte[] key, byte[] value) {
+ ByteBuf keyBuf = Unpooled.wrappedBuffer(key);
+ ByteBuf valueBuf = Unpooled.wrappedBuffer(value);
+ try {
+ // Skip table-prefix, table-id, record-type-prefix
+ keyBuf.readerIndex(4);
+ short topicLen = keyBuf.readShort();
+ assert topicLen == keyBuf.readableBytes();
+ CharSequence topic = keyBuf.readCharSequence(topicLen, StandardCharsets.UTF_8);
+ assert null != topic;
+
+ byte serializationType = valueBuf.readByte();
+ if (SerializationType.JSON == SerializationType.valueOf(serializationType)) {
+ CharSequence json = valueBuf.readCharSequence(valueBuf.readableBytes(), StandardCharsets.UTF_8);
+ TopicConfig topicConfig = JSON.parseObject(json.toString(), TopicConfig.class);
+ assert topicConfig != null;
+ assert topic.equals(topicConfig.getTopicName());
+ return topicConfig;
+ }
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+
+ return null;
+ }
+
+ @Override
+ public synchronized void persist() {
+ try {
+ configStorage.flushWAL();
+ } catch (RocksDBException e) {
+ log.error("Failed to flush WAL", e);
+ }
+ }
+
+ @Override
+ public TopicConfig selectTopicConfig(final String topic) {
+ if (MixAll.isLmq(topic)) {
+ return simpleLmqTopicConfig(topic);
+ }
+ return super.selectTopicConfig(topic);
+ }
+
+ @Override
+ public void updateTopicConfig(final TopicConfig topicConfig) {
+ if (topicConfig == null || MixAll.isLmq(topicConfig.getTopicName())) {
+ return;
+ }
+ super.updateSingleTopicConfigWithoutPersist(topicConfig);
+
+ ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicConfig.getTopicName());
+ ByteBuf valueBuf = ConfigHelper.valueBufOf(topicConfig, SerializationType.JSON);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ writeBatch.put(keyBuf.nioBuffer(), valueBuf.nioBuffer());
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ // fdatasync on core metadata change
+ this.persist();
+ } catch (RocksDBException e) {
+ log.error("Failed to update topic config", e);
+ } finally {
+ keyBuf.release();
+ valueBuf.release();
+ }
+ }
+
+ @Override
+ protected TopicConfig removeTopicConfig(String topicName) {
+ ByteBuf keyBuf = ConfigHelper.keyBufOf(TableId.TOPIC, topicName);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ writeBatch.delete(keyBuf.nioBuffer());
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ ConfigHelper.stampDataVersion(writeBatch, TableId.TOPIC, dataVersion, stateMachineVersion);
+ configStorage.write(writeBatch);
+ } catch (RocksDBException e) {
+ log.error("Failed to delete topic config by topicName={}", topicName, e);
+ } finally {
+ keyBuf.release();
+ }
+ return super.removeTopicConfig(topicName);
+ }
+
+ @Override
+ public boolean containsTopic(String topic) {
+ if (MixAll.isLmq(topic)) {
+ return true;
+ }
+ return super.containsTopic(topic);
+ }
+
+ private TopicConfig simpleLmqTopicConfig(String topic) {
+ return new TopicConfig(topic, 1, 1, PermName.PERM_READ | PermName.PERM_WRITE);
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java
new file mode 100644
index 00000000000..1ea216193c3
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/config/v2/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.config.v2;
+
+/*
+ * Endian: we use network byte order for all integrals, aka, always big endian.
+ *
+ * Unlike v1 config managers, implementations in this package prioritize data integrity and reliability.
+ * As a result,RocksDB write-ahead-log is always on and changes are immediately flushed. Another significant
+ * difference is that heap-based cache is removed because it is not necessary and duplicated to RocksDB
+ * MemTable/BlockCache.
+ */
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
index a1d711cb275..f22f22a12bd 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java
@@ -30,7 +30,7 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.out.BrokerOuterAPI;
@@ -525,7 +525,7 @@ private boolean applyBrokerId() {
return true;
} catch (Exception e) {
- LOGGER.error("fail to apply broker id: {}", e, tempBrokerMetadata.getBrokerId());
+ LOGGER.error("fail to apply broker id: {}", tempBrokerMetadata.getBrokerId(), e);
return false;
}
}
@@ -686,7 +686,7 @@ private void schedulingSyncBrokerMetadata() {
}
/**
- * Scheduling sync controller medata.
+ * Scheduling sync controller metadata.
*/
private boolean schedulingSyncControllerMetadata() {
// Get controller metadata first.
@@ -803,7 +803,10 @@ private void scanAvailableControllerAddresses() {
private void updateControllerAddr() {
if (brokerConfig.isFetchControllerAddrByDnsLookup()) {
- this.controllerAddresses = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr());
+ List adders = brokerOuterAPI.dnsLookupAddressByDomain(this.brokerConfig.getControllerAddr());
+ if (CollectionUtils.isNotEmpty(adders)) {
+ this.controllerAddresses = adders;
+ }
} else {
final String controllerPaths = this.brokerConfig.getControllerAddr();
final String[] controllers = controllerPaths.split(";");
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
index ededaf2c65e..dd37f42b2c5 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java
@@ -25,7 +25,9 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
import org.apache.rocketmq.client.consumer.PullStatus;
@@ -34,7 +36,6 @@
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageConst;
@@ -51,6 +52,7 @@
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
public class EscapeBridge {
protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -99,7 +101,7 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {
try {
messageExt.setWaitStoreMsgOK(false);
- final SendResult sendResult = putMessageToRemoteBroker(messageExt);
+ final SendResult sendResult = putMessageToRemoteBroker(messageExt, null);
return transformSendResult2PutResult(sendResult);
} catch (Exception e) {
LOG.error("sendMessageInFailover to remote failed", e);
@@ -112,7 +114,10 @@ public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {
}
}
- private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) {
+ public SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt, String brokerNameToSend) {
+ if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) { // not remote broker
+ return null;
+ }
final boolean isTransHalfMessage = TransactionalMessageUtil.buildHalfTopic().equals(messageExt.getTopic());
MessageExtBrokerInner messageToPut = messageExt;
if (isTransHalfMessage) {
@@ -125,12 +130,26 @@ private SendResult putMessageToRemoteBroker(MessageExtBrokerInner messageExt) {
return null;
}
- final MessageQueue mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName());
-
- messageToPut.setQueueId(mqSelected.getQueueId());
+ final MessageQueue mqSelected;
+ if (StringUtils.isEmpty(brokerNameToSend)) {
+ mqSelected = topicPublishInfo.selectOneMessageQueue(this.brokerController.getBrokerConfig().getBrokerName());
+ messageToPut.setQueueId(mqSelected.getQueueId());
+ brokerNameToSend = mqSelected.getBrokerName();
+ if (this.brokerController.getBrokerConfig().getBrokerName().equals(brokerNameToSend)) {
+ LOG.warn("putMessageToRemoteBroker failed, remote broker not found. Topic: {}, MsgId: {}, Broker: {}",
+ messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);
+ return null;
+ }
+ } else {
+ mqSelected = new MessageQueue(messageExt.getTopic(), brokerNameToSend, messageExt.getQueueId());
+ }
- final String brokerNameToSend = mqSelected.getBrokerName();
final String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);
+ if (null == brokerAddrToSend) {
+ LOG.warn("putMessageToRemoteBroker failed, remote broker address not found. Topic: {}, MsgId: {}, Broker: {}",
+ messageExt.getTopic(), messageExt.getMsgId(), brokerNameToSend);
+ return null;
+ }
final long beginTimestamp = System.currentTimeMillis();
try {
@@ -178,7 +197,7 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner
producerGroup, SEND_TIMEOUT);
return future.exceptionally(throwable -> null)
- .thenApplyAsync(sendResult -> transformSendResult2PutResult(sendResult), this.defaultAsyncSenderExecutor)
+ .thenApplyAsync(this::transformSendResult2PutResult, this.defaultAsyncSenderExecutor)
.exceptionally(throwable -> transformSendResult2PutResult(null));
} catch (Exception e) {
@@ -192,7 +211,6 @@ public CompletableFuture asyncPutMessage(MessageExtBrokerInner
}
}
-
private String getProducerGroup(MessageExtBrokerInner messageExt) {
if (null == messageExt) {
return this.innerProducerGroupName;
@@ -204,12 +222,29 @@ private String getProducerGroup(MessageExtBrokerInner messageExt) {
return producerGroup;
}
-
public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
BrokerController masterBroker = this.brokerController.peekMasterBroker();
if (masterBroker != null) {
return masterBroker.getMessageStore().putMessage(messageExt);
- } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
+ }
+ try {
+ return asyncRemotePutMessageToSpecificQueue(messageExt).get(SEND_TIMEOUT, TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ LOG.error("Put message to specific queue error", e);
+ return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, true);
+ }
+ }
+
+ public CompletableFuture asyncPutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
+ BrokerController masterBroker = this.brokerController.peekMasterBroker();
+ if (masterBroker != null) {
+ return masterBroker.getMessageStore().asyncPutMessage(messageExt);
+ }
+ return asyncRemotePutMessageToSpecificQueue(messageExt);
+ }
+
+ public CompletableFuture asyncRemotePutMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
+ if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
&& this.brokerController.getBrokerConfig().isEnableRemoteEscape()) {
try {
messageExt.setWaitStoreMsgOK(false);
@@ -218,7 +253,7 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE
List mqs = topicPublishInfo.getMessageQueueList();
if (null == mqs || mqs.isEmpty()) {
- return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
}
String id = messageExt.getTopic() + messageExt.getStoreHost();
@@ -229,19 +264,17 @@ public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageE
String brokerNameToSend = mq.getBrokerName();
String brokerAddrToSend = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInPublish(brokerNameToSend);
- final SendResult sendResult = this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker(
+ return this.brokerController.getBrokerOuterAPI().sendMessageToSpecificBrokerAsync(
brokerAddrToSend, brokerNameToSend,
- messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT);
-
- return transformSendResult2PutResult(sendResult);
+ messageExt, this.getProducerGroup(messageExt), SEND_TIMEOUT).thenCompose(sendResult -> CompletableFuture.completedFuture(transformSendResult2PutResult(sendResult)));
} catch (Exception e) {
LOG.error("sendMessageInFailover to remote failed", e);
- return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
}
} else {
LOG.warn("Put message to specific queue failed, enableSlaveActingMaster={}, enableRemoteEscape={}.",
this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());
- return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null));
}
}
@@ -263,34 +296,35 @@ private PutMessageResult transformSendResult2PutResult(SendResult sendResult) {
}
}
- public Pair getMessage(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) {
+ public Triple getMessage(String topic, long offset, int queueId, String brokerName,
+ boolean deCompressBody) {
return getMessageAsync(topic, offset, queueId, brokerName, deCompressBody).join();
}
- public CompletableFuture> getMessageAsync(String topic, long offset, int queueId, String brokerName, boolean deCompressBody) {
+ // Triple, check info and retry if and only if MessageExt is null
+ public CompletableFuture> getMessageAsync(String topic, long offset,
+ int queueId, String brokerName, boolean deCompressBody) {
MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName);
if (messageStore != null) {
return messageStore.getMessageAsync(innerConsumerGroupName, topic, queueId, offset, 1, null)
.thenApply(result -> {
if (result == null) {
LOG.warn("getMessageResult is null , innerConsumerGroupName {}, topic {}, offset {}, queueId {}", innerConsumerGroupName, topic, offset, queueId);
- return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null);
+ return Triple.of(null, "getMessageResult is null", false); // local store, so no retry
}
List list = decodeMsgList(result, deCompressBody);
if (list == null || list.isEmpty()) {
- LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, result);
- return new Pair<>(result.getStatus(), null);
+ // OFFSET_FOUND_NULL returned by TieredMessageStore indicates exception occurred
+ boolean needRetry = GetMessageStatus.OFFSET_FOUND_NULL.equals(result.getStatus())
+ && messageStore instanceof TieredMessageStore;
+ LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, needRetry {}, result is {}",
+ topic, offset, queueId, needRetry, result);
+ return Triple.of(null, "Can not get msg", needRetry);
}
- return new Pair<>(result.getStatus(), list.get(0));
+ return Triple.of(list.get(0), "", false);
});
} else {
- return getMessageFromRemoteAsync(topic, offset, queueId, brokerName)
- .thenApply(msg -> {
- if (msg == null) {
- return new Pair<>(GetMessageStatus.MESSAGE_WAS_REMOVING, null);
- }
- return new Pair<>(GetMessageStatus.FOUND, msg);
- });
+ return getMessageFromRemoteAsync(topic, offset, queueId, brokerName);
}
}
@@ -322,11 +356,14 @@ protected List decodeMsgList(GetMessageResult getMessageResult, bool
return foundList;
}
- protected MessageExt getMessageFromRemote(String topic, long offset, int queueId, String brokerName) {
+ protected Triple getMessageFromRemote(String topic, long offset, int queueId,
+ String brokerName) {
return getMessageFromRemoteAsync(topic, offset, queueId, brokerName).join();
}
- protected CompletableFuture getMessageFromRemoteAsync(String topic, long offset, int queueId, String brokerName) {
+ // Triple, check info and retry if and only if MessageExt is null
+ protected CompletableFuture> getMessageFromRemoteAsync(String topic,
+ long offset, int queueId, String brokerName) {
try {
String brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);
if (null == brokerAddr) {
@@ -334,23 +371,25 @@ protected CompletableFuture getMessageFromRemoteAsync(String topic,
brokerAddr = this.brokerController.getTopicRouteInfoManager().findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, false);
if (null == brokerAddr) {
- LOG.warn("can't find broker address for topic {}", topic);
- return CompletableFuture.completedFuture(null);
+ LOG.warn("can't find broker address for topic {}, {}", topic, brokerName);
+ return CompletableFuture.completedFuture(Triple.of(null, "brokerAddress not found", true)); // maybe offline temporarily, so need retry
}
}
return this.brokerController.getBrokerOuterAPI().pullMessageFromSpecificBrokerAsync(brokerName,
- brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS)
+ brokerAddr, this.innerConsumerGroupName, topic, queueId, offset, 1, DEFAULT_PULL_TIMEOUT_MILLIS)
.thenApply(pullResult -> {
- if (pullResult.getPullStatus().equals(PullStatus.FOUND) && !pullResult.getMsgFoundList().isEmpty()) {
- return pullResult.getMsgFoundList().get(0);
+ if (pullResult.getLeft() != null
+ && PullStatus.FOUND.equals(pullResult.getLeft().getPullStatus())
+ && CollectionUtils.isNotEmpty(pullResult.getLeft().getMsgFoundList())) {
+ return Triple.of(pullResult.getLeft().getMsgFoundList().get(0), "", false);
}
- return null;
+ return Triple.of(null, pullResult.getMiddle(), pullResult.getRight());
});
} catch (Exception e) {
- LOG.error("Get message from remote failed.", e);
+ LOG.error("Get message from remote failed. {}, {}, {}, {}", topic, offset, queueId, brokerName, e);
}
- return CompletableFuture.completedFuture(null);
+ return CompletableFuture.completedFuture(Triple.of(null, "Get message from remote failed", true)); // need retry
}
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
index 3b6e9dc676e..ce8fdd88579 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java
@@ -16,11 +16,15 @@
*/
package org.apache.rocketmq.broker.latency;
+import java.util.List;
+import java.util.ArrayList;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.AbstractBrokerRunnable;
+import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
@@ -42,13 +46,26 @@ public class BrokerFastFailure {
private volatile long jstackTime = System.currentTimeMillis();
+ private final List, Supplier>> cleanExpiredRequestQueueList = new ArrayList<>();
+
public BrokerFastFailure(final BrokerController brokerController) {
this.brokerController = brokerController;
+ initCleanExpiredRequestQueueList();
this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1,
new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true,
brokerController == null ? null : brokerController.getBrokerConfig()));
}
+ private void initCleanExpiredRequestQueueList() {
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getSendThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getPullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getLitePullThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getHeartbeatThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getEndTransactionThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAckThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue()));
+ cleanExpiredRequestQueueList.add(new Pair<>(this.brokerController.getAdminBrokerThreadPoolQueue(), () -> this.brokerController.getBrokerConfig().getWaitTimeMillsInAdminBrokerQueue()));
+ }
+
public static RequestTask castRunnable(final Runnable runnable) {
try {
if (runnable instanceof FutureTaskExt) {
@@ -98,23 +115,9 @@ private void cleanExpiredRequest() {
}
}
- cleanExpiredRequestInQueue(this.brokerController.getSendThreadPoolQueue(),
- this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue());
-
- cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(),
- this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue());
-
- cleanExpiredRequestInQueue(this.brokerController.getLitePullThreadPoolQueue(),
- this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue());
-
- cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(),
- this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue());
-
- cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this
- .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue());
-
- cleanExpiredRequestInQueue(this.brokerController.getAckThreadPoolQueue(),
- brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue());
+ for (Pair, Supplier> pair : cleanExpiredRequestQueueList) {
+ cleanExpiredRequestInQueue(pair.getObject1(), pair.getObject2().get());
+ }
}
void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, final long maxWaitTimeMillsInQueue) {
@@ -151,6 +154,11 @@ void cleanExpiredRequestInQueue(final BlockingQueue blockingQueue, fin
}
}
+ public synchronized void addCleanExpiredRequestQueue(BlockingQueue cleanExpiredRequestQueue,
+ Supplier maxWaitTimeMillsInQueueSupplier) {
+ cleanExpiredRequestQueueList.add(new Pair<>(cleanExpiredRequestQueue, maxWaitTimeMillsInQueueSupplier));
+ }
+
public void shutdown() {
this.scheduledExecutorService.shutdown();
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
index 88e74fd6e5a..eddaee706a9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
@@ -22,7 +22,6 @@
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
-
public class LmqPullRequestHoldService extends PullRequestHoldService {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -48,8 +47,8 @@ public void checkHoldRequest() {
}
String topic = key.substring(0, idx);
int queueId = Integer.parseInt(key.substring(idx + 1));
- final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
+ final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
LOGGER.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
index e55ed2778ac..9c0ee89e4db 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java
@@ -36,9 +36,12 @@ public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHol
@Override
public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
long msgStoreTime, byte[] filterBitMap, Map properties) {
- this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode,
- msgStoreTime, filterBitMap, properties);
- this.popMessageProcessor.notifyMessageArriving(topic, queueId);
- this.notificationProcessor.notifyMessageArriving(topic, queueId);
+
+ this.pullRequestHoldService.notifyMessageArriving(
+ topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
+ this.popMessageProcessor.notifyMessageArriving(
+ topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
+ this.notificationProcessor.notifyMessageArriving(
+ topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties);
}
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java
new file mode 100644
index 00000000000..2e190e20f92
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopCommandCallback.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.longpolling;
+
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import org.apache.rocketmq.broker.metrics.ConsumerLagCalculator;
+import org.apache.rocketmq.remoting.CommandCallback;
+
+public class PopCommandCallback implements CommandCallback {
+
+ private final BiConsumer> biConsumer;
+
+ private final ConsumerLagCalculator.ProcessGroupInfo info;
+ private final Consumer lagRecorder;
+
+
+ public PopCommandCallback(
+ BiConsumer> biConsumer,
+ ConsumerLagCalculator.ProcessGroupInfo info,
+ Consumer lagRecorder) {
+
+ this.biConsumer = biConsumer;
+ this.info = info;
+ this.lagRecorder = lagRecorder;
+ }
+
+ @Override
+ public void accept() {
+ biConsumer.accept(info, lagRecorder);
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
index a768fe4b9c4..91185fbe94c 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopLongPollingService.java
@@ -19,6 +19,7 @@
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
import io.netty.channel.ChannelHandlerContext;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -31,10 +32,14 @@
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.CommandCallback;
import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.RequestTask;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.ConsumeQueueExt;
+import org.apache.rocketmq.store.MessageFilter;
import static org.apache.rocketmq.broker.longpolling.PollingResult.NOT_POLLING;
import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_FULL;
@@ -42,6 +47,7 @@
import static org.apache.rocketmq.broker.longpolling.PollingResult.POLLING_TIMEOUT;
public class PopLongPollingService extends ServiceThread {
+
private static final Logger POP_LOGGER =
LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
private final BrokerController brokerController;
@@ -147,50 +153,98 @@ public void run() {
}
public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId) {
+ this.notifyMessageArrivingWithRetryTopic(topic, queueId, -1L, null, 0L, null, null);
+ }
+
+ public void notifyMessageArrivingWithRetryTopic(final String topic, final int queueId, long offset,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
String notifyTopic;
if (KeyBuilder.isPopRetryTopicV2(topic)) {
notifyTopic = KeyBuilder.parseNormalTopic(topic);
} else {
notifyTopic = topic;
}
- notifyMessageArriving(notifyTopic, queueId);
+ notifyMessageArriving(notifyTopic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
}
- public void notifyMessageArriving(final String topic, final int queueId) {
+ public void notifyMessageArriving(final String topic, final int queueId, long offset,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
ConcurrentHashMap cids = topicCidMap.get(topic);
if (cids == null) {
return;
}
+ long interval = brokerController.getBrokerConfig().getPopLongPollingForceNotifyInterval();
+ boolean force = interval > 0L && offset % interval == 0L;
for (Map.Entry cid : cids.entrySet()) {
if (queueId >= 0) {
- notifyMessageArriving(topic, cid.getKey(), -1);
+ notifyMessageArriving(topic, -1, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);
}
- notifyMessageArriving(topic, cid.getKey(), queueId);
+ notifyMessageArriving(topic, queueId, cid.getKey(), force, tagsCode, msgStoreTime, filterBitMap, properties);
}
}
- public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) {
+ public boolean notifyMessageArriving(final String topic, final int queueId, final String cid,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
+ return notifyMessageArriving(topic, queueId, cid, false, tagsCode, msgStoreTime, filterBitMap, properties, null);
+ }
+
+ public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
+ return notifyMessageArriving(topic, queueId, cid, force, tagsCode, msgStoreTime, filterBitMap, properties, null);
+ }
+
+ public boolean notifyMessageArriving(final String topic, final int queueId, final String cid, boolean force,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties, CommandCallback callback) {
ConcurrentSkipListSet remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, queueId));
if (remotingCommands == null || remotingCommands.isEmpty()) {
return false;
}
+
PopRequest popRequest = pollRemotingCommands(remotingCommands);
if (popRequest == null) {
return false;
}
+
+ if (!force && popRequest.getMessageFilter() != null && popRequest.getSubscriptionData() != null) {
+ boolean match = popRequest.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
+ new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
+ if (match && properties != null) {
+ match = popRequest.getMessageFilter().isMatchedByCommitLog(null, properties);
+ }
+ if (!match) {
+ remotingCommands.add(popRequest);
+ totalPollingNum.incrementAndGet();
+ return false;
+ }
+ }
+
if (brokerController.getBrokerConfig().isEnablePopLog()) {
- POP_LOGGER.info("lock release , new msg arrive , wakeUp : {}", popRequest);
+ POP_LOGGER.info("lock release, new msg arrive, wakeUp: {}", popRequest);
}
- return wakeUp(popRequest);
+
+ return wakeUp(popRequest, callback);
}
public boolean wakeUp(final PopRequest request) {
+ return wakeUp(request, null);
+ }
+
+ public boolean wakeUp(final PopRequest request, CommandCallback callback) {
if (request == null || !request.complete()) {
return false;
}
+
+ if (callback != null && request.getRemotingCommand() != null) {
+ if (request.getRemotingCommand().getCallbackList() == null) {
+ request.getRemotingCommand().setCallbackList(new ArrayList<>());
+ }
+ request.getRemotingCommand().getCallbackList().add(callback);
+ }
+
if (!request.getCtx().channel().isActive()) {
return false;
}
+
Runnable run = () -> {
try {
final RemotingCommand response = processor.processRequest(request.getCtx(), request.getRemotingCommand());
@@ -209,7 +263,9 @@ public boolean wakeUp(final PopRequest request) {
POP_LOGGER.error("ExecuteRequestWhenWakeup run", e1);
}
};
- this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
+
+ this.brokerController.getPullMessageExecutor().submit(
+ new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
return true;
}
@@ -221,6 +277,11 @@ public boolean wakeUp(final PopRequest request) {
*/
public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,
final PollingHeader requestHeader) {
+ return this.polling(ctx, remotingCommand, requestHeader, null, null);
+ }
+
+ public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand remotingCommand,
+ final PollingHeader requestHeader, SubscriptionData subscriptionData, MessageFilter messageFilter) {
if (requestHeader.getPollTime() <= 0 || this.isStopped()) {
return NOT_POLLING;
}
@@ -234,7 +295,7 @@ public PollingResult polling(final ChannelHandlerContext ctx, RemotingCommand re
}
cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE);
long expired = requestHeader.getBornTime() + requestHeader.getPollTime();
- final PopRequest request = new PopRequest(remotingCommand, ctx, expired);
+ final PopRequest request = new PopRequest(remotingCommand, ctx, expired, subscriptionData, messageFilter);
boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize();
if (isFull) {
POP_LOGGER.info("polling {}, result POLLING_FULL, total:{}", remotingCommand, totalPollingNum.get());
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
index a45bcce9f60..0419dbf637d 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java
@@ -16,28 +16,35 @@
*/
package org.apache.rocketmq.broker.longpolling;
+import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.util.Comparator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-
-import io.netty.channel.Channel;
+import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.store.MessageFilter;
public class PopRequest {
private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE);
private final RemotingCommand remotingCommand;
private final ChannelHandlerContext ctx;
- private final long expired;
private final AtomicBoolean complete = new AtomicBoolean(false);
private final long op = COUNTER.getAndIncrement();
- public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx, long expired) {
+ private final long expired;
+ private final SubscriptionData subscriptionData;
+ private final MessageFilter messageFilter;
+
+ public PopRequest(RemotingCommand remotingCommand, ChannelHandlerContext ctx,
+ long expired, SubscriptionData subscriptionData, MessageFilter messageFilter) {
+
this.ctx = ctx;
this.remotingCommand = remotingCommand;
this.expired = expired;
+ this.subscriptionData = subscriptionData;
+ this.messageFilter = messageFilter;
}
public Channel getChannel() {
@@ -64,6 +71,14 @@ public long getExpired() {
return expired;
}
+ public SubscriptionData getSubscriptionData() {
+ return subscriptionData;
+ }
+
+ public MessageFilter getMessageFilter() {
+ return messageFilter;
+ }
+
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("PopRequest{");
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
index e8da9d0c47c..7dbc9e4fd86 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
@@ -28,6 +28,7 @@
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.ConsumeQueueExt;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class PullRequestHoldService extends ServiceThread {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -103,8 +104,8 @@ protected void checkHoldRequest() {
if (2 == kArray.length) {
String topic = kArray[0];
int queueId = Integer.parseInt(kArray[1]);
- final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
+ final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
log.error(
@@ -131,7 +132,12 @@ public void notifyMessageArriving(final String topic, final int queueId, final l
for (PullRequest request : requestList) {
long newestOffset = maxOffset;
if (newestOffset <= request.getPullFromThisOffset()) {
- newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+ try {
+ newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+ } catch (ConsumeQueueException e) {
+ log.error("Failed tp get max offset in queue", e);
+ continue;
+ }
}
if (newestOffset > request.getPullFromThisOffset()) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
index 5733aa40bac..4b319f12f6f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsConstant.java
@@ -21,12 +21,16 @@ public class BrokerMetricsConstant {
public static final String GAUGE_PROCESSOR_WATERMARK = "rocketmq_processor_watermark";
public static final String GAUGE_BROKER_PERMISSION = "rocketmq_broker_permission";
+ public static final String GAUGE_TOPIC_NUM = "rocketmq_topic_number";
+ public static final String GAUGE_CONSUMER_GROUP_NUM = "rocketmq_consumer_group_number";
public static final String COUNTER_MESSAGES_IN_TOTAL = "rocketmq_messages_in_total";
public static final String COUNTER_MESSAGES_OUT_TOTAL = "rocketmq_messages_out_total";
public static final String COUNTER_THROUGHPUT_IN_TOTAL = "rocketmq_throughput_in_total";
public static final String COUNTER_THROUGHPUT_OUT_TOTAL = "rocketmq_throughput_out_total";
public static final String HISTOGRAM_MESSAGE_SIZE = "rocketmq_message_size";
+ public static final String HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME = "rocketmq_topic_create_execution_time";
+ public static final String HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME = "rocketmq_consumer_group_create_execution_time";
public static final String GAUGE_PRODUCER_CONNECTIONS = "rocketmq_producer_connections";
public static final String GAUGE_CONSUMER_CONNECTIONS = "rocketmq_consumer_connections";
@@ -52,6 +56,7 @@ public class BrokerMetricsConstant {
public static final String LABEL_PROCESSOR = "processor";
public static final String LABEL_TOPIC = "topic";
+ public static final String LABEL_INVOCATION_STATUS = "invocation_status";
public static final String LABEL_IS_RETRY = "is_retry";
public static final String LABEL_IS_SYSTEM = "is_system";
public static final String LABEL_CONSUMER_GROUP = "consumer_group";
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
index fc7e97bda95..d8d94f8e69a 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java
@@ -64,6 +64,7 @@
import org.apache.rocketmq.store.metrics.DefaultStoreMetricsConstant;
import org.slf4j.bridge.SLF4JBridgeHandler;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -80,6 +81,8 @@
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_ROLLBACK_MESSAGES_TOTAL;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_IN_TOTAL;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.COUNTER_THROUGHPUT_OUT_TOTAL;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_TOPIC_NUM;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_GROUP_NUM;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_BROKER_PERMISSION;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_CONNECTIONS;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_CONSUMER_INFLIGHT_MESSAGES;
@@ -92,6 +95,8 @@
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.GAUGE_PRODUCER_CONNECTIONS;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_FINISH_MSG_LATENCY;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_MESSAGE_SIZE;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_AGGREGATION;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CLUSTER_NAME;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
@@ -128,6 +133,9 @@ public class BrokerMetricsManager {
// broker stats metrics
public static ObservableLongGauge processorWatermark = new NopObservableLongGauge();
public static ObservableLongGauge brokerPermission = new NopObservableLongGauge();
+ public static ObservableLongGauge topicNum = new NopObservableLongGauge();
+ public static ObservableLongGauge consumerGroupNum = new NopObservableLongGauge();
+
// request metrics
public static LongCounter messagesInTotal = new NopLongCounter();
@@ -135,6 +143,8 @@ public class BrokerMetricsManager {
public static LongCounter throughputInTotal = new NopLongCounter();
public static LongCounter throughputOutTotal = new NopLongCounter();
public static LongHistogram messageSize = new NopLongHistogram();
+ public static LongHistogram topicCreateExecuteTime = new NopLongHistogram();
+ public static LongHistogram consumerGroupCreateExecuteTime = new NopLongHistogram();
// client connection metrics
public static ObservableLongGauge producerConnection = new NopObservableLongGauge();
@@ -381,6 +391,14 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {
1d * 12 * 60 * 60, //12h
1d * 24 * 60 * 60 //24h
);
+
+ List createTimeBuckets = Arrays.asList(
+ (double) Duration.ofMillis(10).toMillis(), //10ms
+ (double) Duration.ofMillis(100).toMillis(), //100ms
+ (double) Duration.ofSeconds(1).toMillis(), //1s
+ (double) Duration.ofSeconds(3).toMillis(), //3s
+ (double) Duration.ofSeconds(5).toMillis() //5s
+ );
InstrumentSelector messageSizeSelector = InstrumentSelector.builder()
.setType(InstrumentType.HISTOGRAM)
.setName(HISTOGRAM_MESSAGE_SIZE)
@@ -401,6 +419,24 @@ private void registerMetricsView(SdkMeterProviderBuilder providerBuilder) {
SdkMeterProviderUtil.setCardinalityLimit(commitLatencyViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
providerBuilder.registerView(commitLatencySelector, commitLatencyViewBuilder.build());
+ InstrumentSelector createTopicTimeSelector = InstrumentSelector.builder()
+ .setType(InstrumentType.HISTOGRAM)
+ .setName(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)
+ .build();
+ InstrumentSelector createSubGroupTimeSelector = InstrumentSelector.builder()
+ .setType(InstrumentType.HISTOGRAM)
+ .setName(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)
+ .build();
+ ViewBuilder createTopicTimeViewBuilder = View.builder()
+ .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));
+ ViewBuilder createSubGroupTimeViewBuilder = View.builder()
+ .setAggregation(Aggregation.explicitBucketHistogram(createTimeBuckets));
+ // To config the cardinalityLimit for openTelemetry metrics exporting.
+ SdkMeterProviderUtil.setCardinalityLimit(createTopicTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
+ providerBuilder.registerView(createTopicTimeSelector, createTopicTimeViewBuilder.build());
+ SdkMeterProviderUtil.setCardinalityLimit(createSubGroupTimeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
+ providerBuilder.registerView(createSubGroupTimeSelector, createSubGroupTimeViewBuilder.build());
+
for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) {
ViewBuilder viewBuilder = selectorViewPair.getObject2();
SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit());
@@ -459,6 +495,16 @@ private void initStatsMetrics() {
.setDescription("Broker permission")
.ofLongs()
.buildWithCallback(measurement -> measurement.record(brokerConfig.getBrokerPermission(), newAttributesBuilder().build()));
+
+ topicNum = brokerMeter.gaugeBuilder(GAUGE_TOPIC_NUM)
+ .setDescription("Active topic number")
+ .ofLongs()
+ .buildWithCallback(measurement -> measurement.record(brokerController.getTopicConfigManager().getTopicConfigTable().size(), newAttributesBuilder().build()));
+
+ consumerGroupNum = brokerMeter.gaugeBuilder(GAUGE_CONSUMER_GROUP_NUM)
+ .setDescription("Active subscription group number")
+ .ofLongs()
+ .buildWithCallback(measurement -> measurement.record(brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().size(), newAttributesBuilder().build()));
}
private void initRequestMetrics() {
@@ -482,6 +528,18 @@ private void initRequestMetrics() {
.setDescription("Incoming messages size")
.ofLongs()
.build();
+
+ topicCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_TOPIC_CREATE_EXECUTE_TIME)
+ .setDescription("The distribution of create topic time")
+ .ofLongs()
+ .setUnit("milliseconds")
+ .build();
+
+ consumerGroupCreateExecuteTime = brokerMeter.histogramBuilder(HISTOGRAM_CONSUMER_GROUP_CREATE_EXECUTE_TIME)
+ .setDescription("The distribution of create subscription time")
+ .ofLongs()
+ .setUnit("milliseconds")
+ .build();
}
private void initConnectionMetrics() {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
index 1930d0dfcb6..1b898f95de3 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
@@ -26,6 +26,8 @@
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+import org.apache.rocketmq.broker.longpolling.PopCommandCallback;
+import org.apache.rocketmq.broker.longpolling.PopLongPollingService;
import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
import org.apache.rocketmq.broker.processor.PopBufferMergeService;
import org.apache.rocketmq.broker.processor.PopInflightMessageCounter;
@@ -48,8 +50,10 @@
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.store.DefaultMessageFilter;
import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class ConsumerLagCalculator {
+
private final BrokerConfig brokerConfig;
private final TopicConfigManager topicConfigManager;
private final ConsumerManager consumerManager;
@@ -58,6 +62,7 @@ public class ConsumerLagCalculator {
private final SubscriptionGroupManager subscriptionGroupManager;
private final MessageStore messageStore;
private final PopBufferMergeService popBufferMergeService;
+ private final PopLongPollingService popLongPollingService;
private final PopInflightMessageCounter popInflightMessageCounter;
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -71,10 +76,11 @@ public ConsumerLagCalculator(BrokerController brokerController) {
this.subscriptionGroupManager = brokerController.getSubscriptionGroupManager();
this.messageStore = brokerController.getMessageStore();
this.popBufferMergeService = brokerController.getPopMessageProcessor().getPopBufferMergeService();
+ this.popLongPollingService = brokerController.getPopMessageProcessor().getPopLongPollingService();
this.popInflightMessageCounter = brokerController.getPopInflightMessageCounter();
}
- private static class ProcessGroupInfo {
+ public static class ProcessGroupInfo {
public String group;
public String topic;
public boolean isPop;
@@ -210,16 +216,32 @@ public void calculateLag(Consumer lagRecorder) {
return;
}
- CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);
+ if (info.isPop && brokerConfig.isEnableNotifyBeforePopCalculateLag()) {
+ if (popLongPollingService.notifyMessageArriving(info.topic, -1, info.group,
+ true, null, 0, null, null, new PopCommandCallback(this::calculate, info, lagRecorder))) {
+ return;
+ }
+ }
+
+ calculate(info, lagRecorder);
+ });
+ }
+ public void calculate(ProcessGroupInfo info, Consumer lagRecorder) {
+ CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);
+ try {
Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop);
if (lag != null) {
result.lag = lag.getObject1();
result.earliestUnconsumedTimestamp = lag.getObject2();
}
lagRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get lag stats", e);
+ }
- if (info.isPop) {
+ if (info.isPop) {
+ try {
Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true);
result = new CalculateLagResult(info.group, info.topic, true);
@@ -228,29 +250,39 @@ public void calculateLag(Consumer lagRecorder) {
result.earliestUnconsumedTimestamp = retryLag.getObject2();
}
lagRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get lag stats", e);
}
- });
+ }
}
public void calculateInflight(Consumer inflightRecorder) {
processAllGroup(info -> {
CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false);
- Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
- if (inFlight != null) {
- result.inFlight = inFlight.getObject1();
- result.earliestUnPulledTimestamp = inFlight.getObject2();
+ try {
+ Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
+ if (inFlight != null) {
+ result.inFlight = inFlight.getObject1();
+ result.earliestUnPulledTimestamp = inFlight.getObject2();
+ }
+ inflightRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get inflight message stats", e);
}
- inflightRecorder.accept(result);
if (info.isPop) {
- Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
+ try {
+ Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
- result = new CalculateInflightResult(info.group, info.topic, true);
- if (retryInFlight != null) {
- result.inFlight = retryInFlight.getObject1();
- result.earliestUnPulledTimestamp = retryInFlight.getObject2();
+ result = new CalculateInflightResult(info.group, info.topic, true);
+ if (retryInFlight != null) {
+ result.inFlight = retryInFlight.getObject1();
+ result.earliestUnPulledTimestamp = retryInFlight.getObject2();
+ }
+ inflightRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get inflight message stats", e);
}
- inflightRecorder.accept(result);
}
});
}
@@ -259,20 +291,28 @@ public void calculateAvailable(Consumer availableRecor
processAllGroup(info -> {
CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false);
- result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
- availableRecorder.accept(result);
+ try {
+ result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
+ availableRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get available message count", e);
+ }
+
if (info.isPop) {
- long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
-
- result = new CalculateAvailableResult(info.group, info.topic, true);
- result.available = retryAvailable;
- availableRecorder.accept(result);
+ try {
+ long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
+ result = new CalculateAvailableResult(info.group, info.topic, true);
+ result.available = retryAvailable;
+ availableRecorder.accept(result);
+ } catch (ConsumeQueueException e) {
+ LOGGER.error("Failed to get available message count", e);
+ }
}
});
}
- public Pair getConsumerLagStats(String group, String topic, boolean isPop) {
+ public Pair getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
long earliestUnconsumedTimestamp = Long.MAX_VALUE;
@@ -295,10 +335,14 @@ public Pair getConsumerLagStats(String group, String topic, boolean
earliestUnconsumedTimestamp = 0L;
}
+ LOGGER.debug("GetConsumerLagStats, topic={}, group={}, lag={}, latency={}", topic, group, total,
+ earliestUnconsumedTimestamp > 0 ? System.currentTimeMillis() - earliestUnconsumedTimestamp : 0);
+
return new Pair<>(total, earliestUnconsumedTimestamp);
}
- public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop) {
+ public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop)
+ throws ConsumeQueueException {
long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
if (brokerOffset < 0) {
brokerOffset = 0;
@@ -329,7 +373,7 @@ public Pair getConsumerLagStats(String group, String topic, int queu
return new Pair<>(lag, consumerStoreTimeStamp);
}
- public Pair getInFlightMsgStats(String group, String topic, boolean isPop) {
+ public Pair getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
long earliestUnPulledTimestamp = Long.MAX_VALUE;
@@ -355,7 +399,8 @@ public Pair getInFlightMsgStats(String group, String topic, boolean
return new Pair<>(total, earliestUnPulledTimestamp);
}
- public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) {
+ public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop)
+ throws ConsumeQueueException {
if (isPop) {
long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId);
long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);
@@ -384,7 +429,7 @@ public Pair getInFlightMsgStats(String group, String topic, int queu
return new Pair<>(inflight, pullStoreTimeStamp);
}
- public long getAvailableMsgCount(String group, String topic, boolean isPop) {
+ public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
if (group == null || topic == null) {
@@ -403,7 +448,8 @@ public long getAvailableMsgCount(String group, String topic, boolean isPop) {
return total;
}
- public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) {
+ public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop)
+ throws ConsumeQueueException {
long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
if (brokerOffset < 0) {
brokerOffset = 0;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java
new file mode 100644
index 00000000000..c7501e53d96
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/InvocationStatus.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.metrics;
+
+public enum InvocationStatus {
+ SUCCESS("success"),
+ FAILURE("failure");
+
+ private final String name;
+
+ InvocationStatus(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
\ No newline at end of file
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
index 2de220da166..6e87cb0b69e 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
@@ -39,8 +39,11 @@
import org.apache.rocketmq.common.metrics.NopLongCounter;
import org.apache.rocketmq.common.metrics.NopLongHistogram;
import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
+import org.apache.rocketmq.logging.org.slf4j.Logger;
+import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;
@@ -57,6 +60,7 @@
import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE;
public class PopMetricsManager {
+ private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class);
public static Supplier attributesBuilderSupplier;
private static LongHistogram popBufferScanTimeConsume = new NopLongHistogram();
@@ -138,9 +142,13 @@ private static void calculatePopReviveLatency(BrokerController brokerController,
ObservableLongMeasurement measurement) {
PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
for (PopReviveService popReviveService : popReviveServices) {
- measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
- .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
- .build());
+ try {
+ measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
+ .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
+ .build());
+ } catch (ConsumeQueueException e) {
+ log.error("Failed to get revive behind duration", e);
+ }
}
}
@@ -148,9 +156,13 @@ private static void calculatePopReviveLag(BrokerController brokerController,
ObservableLongMeasurement measurement) {
PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
for (PopReviveService popReviveService : popReviveServices) {
- measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
- .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
- .build());
+ try {
+ measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
+ .put(LABEL_QUEUE_ID, popReviveService.getQueueId())
+ .build());
+ } catch (ConsumeQueueException e) {
+ log.error("Failed to get revive behind message count", e);
+ }
}
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
index 9896735dd1c..79bb0c771d6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
@@ -25,6 +25,7 @@
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
/**
* manage the offset of broadcast.
@@ -72,7 +73,7 @@ public void updateOffset(String topic, String group, int queueId, long offset, S
* @return -1 means no init offset, use the queueOffset in pullRequestHeader
*/
public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset,
- boolean fromProxy) {
+ boolean fromProxy) throws ConsumeQueueException {
BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId));
if (broadcastOffsetData == null) {
@@ -84,29 +85,26 @@ public Long queryInitOffset(String topic, String groupId, int queueId, String cl
}
final AtomicLong offset = new AtomicLong(-1L);
- broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdK, offsetStore) -> {
- if (offsetStore == null) {
- offsetStore = new BroadcastTimedOffsetStore(fromProxy);
- }
+ BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId);
+ if (offsetStore == null) {
+ offsetStore = new BroadcastTimedOffsetStore(fromProxy);
+ broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore);
+ }
- if (offsetStore.fromProxy && requestOffset < 0) {
- // when from proxy and requestOffset is -1
- // means proxy need a init offset to pull message
+ if (offsetStore.fromProxy && requestOffset < 0) {
+ // when from proxy and requestOffset is -1
+ // means proxy need a init offset to pull message
+ offset.set(getOffset(offsetStore, topic, groupId, queueId));
+ } else {
+ if (offsetStore.fromProxy != fromProxy) {
offset.set(getOffset(offsetStore, topic, groupId, queueId));
- return offsetStore;
- }
-
- if (offsetStore.fromProxy == fromProxy) {
- return offsetStore;
}
-
- offset.set(getOffset(offsetStore, topic, groupId, queueId));
- return offsetStore;
- });
+ }
return offset.get();
}
- private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) {
+ private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId)
+ throws ConsumeQueueException {
long storeOffset = -1;
if (offsetStore != null) {
storeOffset = offsetStore.offsetStore.readOffset(queueId);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
index 21f20dde325..ea46f1d8a1f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java
@@ -31,6 +31,7 @@
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
@@ -42,7 +43,7 @@ public class ConsumerOffsetManager extends ConfigManager {
protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
public static final String TOPIC_GROUP_SEPARATOR = "@";
- private DataVersion dataVersion = new DataVersion();
+ protected DataVersion dataVersion = new DataVersion();
protected ConcurrentMap> offsetTable =
new ConcurrentHashMap<>(512);
@@ -373,6 +374,25 @@ public void setDataVersion(DataVersion dataVersion) {
this.dataVersion = dataVersion;
}
+ public boolean loadDataVersion() {
+ String fileName = null;
+ try {
+ fileName = this.configFilePath();
+ String jsonString = MixAll.file2String(fileName);
+ if (jsonString != null) {
+ ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);
+ if (obj != null) {
+ this.dataVersion = obj.dataVersion;
+ }
+ LOG.info("load consumer offset dataVersion success,{},{} ", fileName, jsonString);
+ }
+ return true;
+ } catch (Exception e) {
+ LOG.error("load consumer offset dataVersion failed " + fileName, e);
+ return false;
+ }
+ }
+
public void removeOffset(final String group) {
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
index ce70b1a820f..53e9e2e0634 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/LmqConsumerOffsetManager.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.broker.offset;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -110,4 +111,25 @@ public ConcurrentHashMap getLmqOffsetTable() {
public void setLmqOffsetTable(ConcurrentHashMap lmqOffsetTable) {
this.lmqOffsetTable = lmqOffsetTable;
}
+
+ @Override
+ public void removeOffset(String group) {
+ if (!MixAll.isLmq(group)) {
+ super.removeOffset(group);
+ return;
+ }
+ Iterator> it = this.lmqOffsetTable.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry next = it.next();
+ String topicAtGroup = next.getKey();
+ if (topicAtGroup.contains(group)) {
+ String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+ if (arrays.length == 2 && group.equals(arrays[1])) {
+ it.remove();
+ removeConsumerOffset(topicAtGroup);
+ LOG.warn("clean lmq group offset {}", topicAtGroup);
+ }
+ }
+ }
+ }
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java
deleted file mode 100644
index d0faa661406..00000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBLmqConsumerOffsetManager.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.broker.offset;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
-
-public class RocksDBLmqConsumerOffsetManager extends RocksDBConsumerOffsetManager {
- private ConcurrentHashMap lmqOffsetTable = new ConcurrentHashMap<>(512);
-
- public RocksDBLmqConsumerOffsetManager(BrokerController brokerController) {
- super(brokerController);
- }
-
- @Override
- public long queryOffset(final String group, final String topic, final int queueId) {
- if (!MixAll.isLmq(group)) {
- return super.queryOffset(group, topic, queueId);
- }
- // topic@group
- String key = topic + TOPIC_GROUP_SEPARATOR + group;
- Long offset = lmqOffsetTable.get(key);
- if (offset != null) {
- return offset;
- }
- return -1;
- }
-
- @Override
- public Map queryOffset(final String group, final String topic) {
- if (!MixAll.isLmq(group)) {
- return super.queryOffset(group, topic);
- }
- Map map = new HashMap<>();
- // topic@group
- String key = topic + TOPIC_GROUP_SEPARATOR + group;
- Long offset = lmqOffsetTable.get(key);
- if (offset != null) {
- map.put(0, offset);
- }
- return map;
- }
-
- @Override
- public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,
- final long offset) {
- if (!MixAll.isLmq(group)) {
- super.commitOffset(clientHost, group, topic, queueId, offset);
- return;
- }
- // topic@group
- String key = topic + TOPIC_GROUP_SEPARATOR + group;
- lmqOffsetTable.put(key, offset);
- }
-
- @Override
- public String encode() {
- return this.encode(false);
- }
-
- @Override
- public void decode(String jsonString) {
- if (jsonString != null) {
- RocksDBLmqConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, RocksDBLmqConsumerOffsetManager.class);
- if (obj != null) {
- super.setOffsetTable(obj.getOffsetTable());
- this.lmqOffsetTable = obj.lmqOffsetTable;
- }
- }
- }
-
- @Override
- public String encode(final boolean prettyFormat) {
- return RemotingSerializable.toJson(this, prettyFormat);
- }
-
- public ConcurrentHashMap getLmqOffsetTable() {
- return lmqOffsetTable;
- }
-
- public void setLmqOffsetTable(ConcurrentHashMap lmqOffsetTable) {
- this.lmqOffsetTable = lmqOffsetTable;
- }
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
index d1cdb297fed..83edd88408a 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
@@ -31,6 +31,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
import org.apache.rocketmq.acl.common.AclClientRPCHook;
import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.auth.config.AuthConfig;
@@ -336,7 +337,7 @@ public void sendHeartbeatViaDataVersion(
final DataVersion dataVersion,
final boolean isInBrokerContainer) {
List nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
- if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+ if (nameServerAddressList != null && !nameServerAddressList.isEmpty()) {
final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
requestHeader.setBrokerAddr(brokerAddr);
requestHeader.setBrokerName(brokerName);
@@ -405,7 +406,7 @@ public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
- ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
+ ExchangeHAInfoResponseHeader responseHeader = response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());
}
default:
@@ -574,8 +575,7 @@ private RegisterBrokerResult registerBroker(
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
- RegisterBrokerResponseHeader responseHeader =
- (RegisterBrokerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class);
+ RegisterBrokerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class);
RegisterBrokerResult result = new RegisterBrokerResult();
result.setMasterAddr(responseHeader.getMasterAddr());
result.setHaServerAddr(responseHeader.getHaServerAddr());
@@ -725,7 +725,7 @@ public void run0() {
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
QueryDataVersionResponseHeader queryDataVersionResponseHeader =
- (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
+ response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
changed = queryDataVersionResponseHeader.getChanged();
byte[] body = response.getBody();
if (body != null) {
@@ -887,7 +887,7 @@ public long getMaxOffset(final String addr, final String topic, final int queueI
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
- GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
+ GetMaxOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
return responseHeader.getOffset();
}
@@ -909,7 +909,7 @@ public long getMinOffset(final String addr, final String topic, final int queueI
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
- GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
+ GetMinOffsetResponseHeader responseHeader = response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
return responseHeader.getOffset();
}
@@ -1096,8 +1096,7 @@ private SendResult processSendResponse(
break;
}
if (sendStatus != null) {
- SendMessageResponseHeader responseHeader =
- (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+ SendMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
//If namespace not null , reset Topic without namespace.
String topic = msg.getTopic();
@@ -1270,7 +1269,7 @@ public Pair> brokerElect(String controllerA
// Only record success response.
case CONTROLLER_MASTER_STILL_EXIST:
case SUCCESS:
- final ElectMasterResponseHeader responseHeader = (ElectMasterResponseHeader) response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);
+ final ElectMasterResponseHeader responseHeader = response.decodeCommandCustomHeader(ElectMasterResponseHeader.class);
final ElectMasterResponseBody responseBody = RemotingSerializable.decode(response.getBody(), ElectMasterResponseBody.class);
return new Pair<>(responseHeader, responseBody.getSyncStateSet());
}
@@ -1285,7 +1284,7 @@ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, f
final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
assert response != null;
if (response.getCode() == SUCCESS) {
- return (GetNextBrokerIdResponseHeader) response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);
+ return response.decodeCommandCustomHeader(GetNextBrokerIdResponseHeader.class);
}
throw new MQBrokerException(response.getCode(), response.getRemark());
}
@@ -1297,7 +1296,7 @@ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final
final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
assert response != null;
if (response.getCode() == SUCCESS) {
- return (ApplyBrokerIdResponseHeader) response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);
+ return response.decodeCommandCustomHeader(ApplyBrokerIdResponseHeader.class);
}
throw new MQBrokerException(response.getCode(), response.getRemark());
}
@@ -1310,7 +1309,7 @@ public Pair> registerBrokerT
final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000);
assert response != null;
if (response.getCode() == SUCCESS) {
- RegisterBrokerToControllerResponseHeader responseHeader = (RegisterBrokerToControllerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);
+ RegisterBrokerToControllerResponseHeader responseHeader = response.decodeCommandCustomHeader(RegisterBrokerToControllerResponseHeader.class);
Set syncStateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class).getSyncStateSet();
return new Pair<>(responseHeader, syncStateSet);
}
@@ -1328,7 +1327,7 @@ public Pair getReplicaInfo(final Str
assert response != null;
switch (response.getCode()) {
case SUCCESS: {
- final GetReplicaInfoResponseHeader header = (GetReplicaInfoResponseHeader) response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);
+ final GetReplicaInfoResponseHeader header = response.decodeCommandCustomHeader(GetReplicaInfoResponseHeader.class);
assert response.getBody() != null;
final SyncStateSet stateSet = RemotingSerializable.decode(response.getBody(), SyncStateSet.class);
return new Pair<>(header, stateSet);
@@ -1380,7 +1379,8 @@ public void run0() {
});
}
- public CompletableFuture pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr,
+ // Triple, should check info and retry if and only if PullResult is null
+ public CompletableFuture> pullMessageFromSpecificBrokerAsync(String brokerName, String brokerAddr,
String consumerGroup, String topic, int queueId, long offset,
int maxNums, long timeoutMillis) throws RemotingException, InterruptedException {
PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
@@ -1399,7 +1399,7 @@ public CompletableFuture pullMessageFromSpecificBrokerAsync(String b
requestHeader.setBrokerName(brokerName);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
- CompletableFuture pullResultFuture = new CompletableFuture<>();
+ CompletableFuture> pullResultFuture = new CompletableFuture<>();
this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() {
@Override
public void operationComplete(ResponseFuture responseFuture) {
@@ -1411,15 +1411,16 @@ public void operationSucceed(RemotingCommand response) {
try {
PullResultExt pullResultExt = processPullResponse(response, brokerAddr);
processPullResult(pullResultExt, brokerName, queueId);
- pullResultFuture.complete(pullResultExt);
+ pullResultFuture.complete(Triple.of(pullResultExt, pullResultExt.getPullStatus().name(), false)); // found or not found really, so no retry
} catch (Exception e) {
- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
+ // retry when NO_PERMISSION, SUBSCRIPTION_GROUP_NOT_EXIST etc. even when TOPIC_NOT_EXIST
+ pullResultFuture.complete(Triple.of(null, "Response Code:" + response.getCode(), true));
}
}
@Override
public void operationFail(Throwable throwable) {
- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>()));
+ pullResultFuture.complete(Triple.of(null, throwable.getMessage(), true));
}
});
return pullResultFuture;
@@ -1447,8 +1448,7 @@ private PullResultExt processPullResponse(
throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
}
- PullMessageResponseHeader responseHeader =
- (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
+ PullMessageResponseHeader responseHeader = response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(),
responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta());
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
index ba2d1b5f320..39befedaa22 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
@@ -467,7 +467,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,
TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(requestHeader.getTopic());
if (!result.isValid()) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(result.getRemark());
return response;
}
@@ -522,7 +522,7 @@ protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,
RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
LOGGER.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
index 9a56498632f..043ef13f5a9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
@@ -20,6 +20,7 @@
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.util.BitSet;
+import java.nio.charset.StandardCharsets;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.metrics.PopMetricsManager;
import org.apache.rocketmq.common.KeyBuilder;
@@ -30,7 +31,6 @@
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -45,6 +45,7 @@
import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
@@ -98,7 +99,7 @@ public boolean isPopReviveServiceRunning() {
@Override
public RemotingCommand processRequest(final ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ RemotingCommand request) throws RemotingCommandException {
return this.processRequest(ctx.channel(), request, true);
}
@@ -108,7 +109,7 @@ public boolean rejectRequest() {
}
private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
- boolean brokerAllowSuspend) throws RemotingCommandException {
+ boolean brokerAllowSuspend) throws RemotingCommandException {
AckMessageRequestHeader requestHeader;
BatchAckMessageRequestBody reqBody = null;
final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
@@ -126,7 +127,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
- requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+ requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
POP_LOGGER.warn(errorInfo);
response.setCode(ResponseCode.MESSAGE_ILLEGAL);
response.setRemark(errorInfo);
@@ -134,10 +135,15 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
}
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
- long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+ long maxOffset;
+ try {
+ maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset", e);
+ }
if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d",
- requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
+ requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
POP_LOGGER.warn(errorInfo);
response.setCode(ResponseCode.NO_MESSAGE);
response.setRemark(errorInfo);
@@ -165,7 +171,8 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
return response;
}
- private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, final RemotingCommand response, final Channel channel, String brokerName) {
+ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,
+ final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {
String[] extraInfo;
String consumeGroup, topic;
int qId, rqId;
@@ -205,7 +212,12 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
invisibleTime = batchAck.getInvisibleTime();
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId);
- long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
+ long maxOffset;
+ try {
+ maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
if (minOffset == -1 || maxOffset == -1) {
POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck);
return;
@@ -253,7 +265,7 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(reviveTopic);
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(rqId);
if (ackMsg instanceof BatchAckMsg) {
msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
@@ -268,18 +280,36 @@ private void appendAck(final AckMessageRequestHeader requestHeader, final BatchA
msgInner.setDeliverTimeMs(popTime + invisibleTime);
msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+ int finalAckCount = ackCount;
+ this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+ handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);
+ }).exceptionally(throwable -> {
+ handlePutMessageResult(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, null, false),
+ ackMsg, topic, consumeGroup, popTime, qId, finalAckCount);
+ POP_LOGGER.error("put ack msg error ", throwable);
+ return null;
+ });
+ } else {
+ PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ handlePutMessageResult(putMessageResult, ackMsg, topic, consumeGroup, popTime, qId, ackCount);
+ }
+ }
+
+ private void handlePutMessageResult(PutMessageResult putMessageResult, AckMsg ackMsg, String topic,
+ String consumeGroup, long popTime, int qId, int ackCount) {
if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
POP_LOGGER.error("put ack msg error:" + putMessageResult);
}
PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount);
}
- protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) {
+ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime,
+ long invisibleTime, Channel channel, RemotingCommand response) {
String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId;
long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId);
if (ackOffset < oldOffset) {
@@ -297,15 +327,12 @@ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOf
qId, ackOffset,
popTime);
if (nextOffset > -1) {
- if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(
- topic, consumeGroup, qId)) {
- this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
- consumeGroup, topic, qId, nextOffset);
+ if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset(topic, consumeGroup, qId)) {
+ this.brokerController.getConsumerOffsetManager().commitOffset(
+ channel.remoteAddress().toString(), consumeGroup, topic, qId, nextOffset);
}
- if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic,
- consumeGroup, qId, invisibleTime)) {
- this.brokerController.getPopMessageProcessor().notifyMessageArriving(
- topic, consumeGroup, qId);
+ if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, consumeGroup, qId, invisibleTime)) {
+ this.brokerController.getPopMessageProcessor().notifyMessageArriving(topic, qId, consumeGroup);
}
} else if (nextOffset == -1) {
String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s",
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
index 78a5ba92eed..ffac714c1ba 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
@@ -21,6 +21,7 @@
import com.google.common.collect.Sets;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
+import io.opentelemetry.api.common.Attributes;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
@@ -37,10 +38,14 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.acl.common.AclException;
import org.apache.rocketmq.acl.plain.PlainAccessValidator;
import org.apache.rocketmq.auth.authentication.enums.UserType;
import org.apache.rocketmq.auth.authentication.exception.AuthenticationException;
@@ -58,15 +63,21 @@
import org.apache.rocketmq.broker.controller.ReplicasManager;
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
+import org.apache.rocketmq.broker.metrics.InvocationStatus;
import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil;
+import org.apache.rocketmq.common.BoundaryType;
import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.CheckRocksdbCqWriteResult;
import org.apache.rocketmq.common.KeyBuilder;
import org.apache.rocketmq.common.LockCallback;
import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.common.TopicAttributes;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UnlockCallback;
import org.apache.rocketmq.common.UtilAll;
@@ -113,6 +124,7 @@
import org.apache.rocketmq.remoting.protocol.body.ConsumeQueueData;
import org.apache.rocketmq.remoting.protocol.body.ConsumeStatsList;
import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection;
+import org.apache.rocketmq.remoting.protocol.body.CreateTopicListRequestBody;
import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache;
import org.apache.rocketmq.remoting.protocol.body.GroupList;
import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
@@ -127,11 +139,13 @@
import org.apache.rocketmq.remoting.protocol.body.QuerySubscriptionResponseBody;
import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.remoting.protocol.body.ResetOffsetBody;
+import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupList;
import org.apache.rocketmq.remoting.protocol.body.SyncStateSet;
import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
import org.apache.rocketmq.remoting.protocol.body.TopicList;
import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody;
import org.apache.rocketmq.remoting.protocol.body.UserInfo;
+import org.apache.rocketmq.remoting.protocol.header.CheckRocksdbCqWriteProgressRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.CloneGroupOffsetRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.CreateAccessConfigRequestHeader;
@@ -204,8 +218,12 @@
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.RocksDBMessageStore;
import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.StoreType;
import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
+import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.queue.ReferredIterator;
@@ -213,12 +231,15 @@
import org.apache.rocketmq.store.timer.TimerMessageStore;
import org.apache.rocketmq.store.util.LibC;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_INVOCATION_STATUS;
+import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
public class AdminBrokerProcessor implements NettyRequestProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
protected final BrokerController brokerController;
protected Set configBlackList = new HashSet<>();
+ private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
public AdminBrokerProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
@@ -239,6 +260,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
switch (request.getCode()) {
case RequestCode.UPDATE_AND_CREATE_TOPIC:
return this.updateAndCreateTopic(ctx, request);
+ case RequestCode.UPDATE_AND_CREATE_TOPIC_LIST:
+ return this.updateAndCreateTopicList(ctx, request);
case RequestCode.DELETE_TOPIC_IN_BROKER:
return this.deleteTopic(ctx, request);
case RequestCode.GET_ALL_TOPIC_CONFIG:
@@ -275,6 +298,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
return this.unlockBatchMQ(ctx, request);
case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP:
return this.updateAndCreateSubscriptionGroup(ctx, request);
+ case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP_LIST:
+ return this.updateAndCreateSubscriptionGroupList(ctx, request);
case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG:
return this.getAllSubscriptionGroup(ctx, request);
case RequestCode.DELETE_SUBSCRIPTIONGROUP:
@@ -329,6 +354,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
return fetchAllConsumeStatsInBroker(ctx, request);
case RequestCode.QUERY_CONSUME_QUEUE:
return queryConsumeQueue(ctx, request);
+ case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS:
+ return this.checkRocksdbCqWriteProgress(ctx, request);
case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN:
return this.updateAndGetGroupForbidden(ctx, request);
case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG:
@@ -398,7 +425,7 @@ private RemotingCommand getSubscriptionGroup(ChannelHandlerContext ctx,
SubscriptionGroupConfig groupConfig = this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(requestHeader.getGroup());
if (groupConfig == null) {
LOGGER.error("No group in this broker, client: {} group: {}", ctx.channel().remoteAddress(), requestHeader.getGroup());
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
response.setRemark("No group in this broker");
return response;
}
@@ -448,6 +475,24 @@ private RemotingCommand updateAndGetGroupForbidden(ChannelHandlerContext ctx, Re
return response;
}
+ private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, RemotingCommand request) {
+ CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();
+ result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_IN_PROGRESS.getValue());
+ Runnable runnable = () -> {
+ try {
+ CheckRocksdbCqWriteResult checkResult = doCheckRocksdbCqWriteProgress(ctx, request);
+ LOGGER.info("checkRocksdbCqWriteProgress result: {}", JSON.toJSONString(checkResult));
+ } catch (Exception e) {
+ LOGGER.error("checkRocksdbCqWriteProgress error", e);
+ }
+ };
+ asyncExecuteWorker.submit(runnable);
+ RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setBody(JSON.toJSONBytes(result));
+ return response;
+ }
+
@Override
public boolean rejectRequest() {
return false;
@@ -455,6 +500,7 @@ public boolean rejectRequest() {
private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
+ long startTime = System.currentTimeMillis();
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final CreateTopicRequestHeader requestHeader =
(CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class);
@@ -464,58 +510,154 @@ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext
String topic = requestHeader.getTopic();
- TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
- if (!result.isValid()) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
- response.setRemark(result.getRemark());
- return response;
- }
- if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
- if (TopicValidator.isSystemTopic(topic)) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
- response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+ long executionTime;
+ try {
+ TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
+ if (!result.isValid()) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark(result.getRemark());
return response;
}
- }
+ if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
+ if (TopicValidator.isSystemTopic(topic)) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+ return response;
+ }
+ }
- TopicConfig topicConfig = new TopicConfig(topic);
- topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
- topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());
- topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());
- topicConfig.setPerm(requestHeader.getPerm());
- topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());
- topicConfig.setOrder(requestHeader.getOrder());
- String attributesModification = requestHeader.getAttributes();
- topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));
+ TopicConfig topicConfig = new TopicConfig(topic);
+ topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
+ topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums());
+ topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum());
+ topicConfig.setPerm(requestHeader.getPerm());
+ topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag());
+ topicConfig.setOrder(requestHeader.getOrder());
+ String attributesModification = requestHeader.getAttributes();
+ topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification));
+
+ if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {
+ // Get attribute by key with prefix sign
+ String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();
+ String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);
+ if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark("MIXED message type is not supported.");
+ return response;
+ }
+ }
+
+ if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
+ LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}",
+ requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+ response.setCode(ResponseCode.SUCCESS);
+ return response;
+ }
- if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED
- && !brokerController.getBrokerConfig().isEnableMixedMessageType()) {
+ this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
+ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
+ this.brokerController.registerSingleTopicAll(topicConfig);
+ } else {
+ this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion());
+ }
+ response.setCode(ResponseCode.SUCCESS);
+ } catch (Exception e) {
+ LOGGER.error("Update / create topic failed for [{}]", request, e);
response.setCode(ResponseCode.SYSTEM_ERROR);
- response.setRemark("MIXED message type is not supported.");
+ response.setRemark(e.getMessage());
return response;
+ } finally {
+ executionTime = System.currentTimeMillis() - startTime;
+ InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+ InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+ Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+ .put(LABEL_INVOCATION_STATUS, status.getName())
+ .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topic))
+ .build();
+ BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes);
}
+ LOGGER.info("executionTime of create topic:{} is {} ms", topic, executionTime);
+ return response;
+ }
- if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
- LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}",
- requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
- response.setCode(ResponseCode.SUCCESS);
- return response;
+ private synchronized RemotingCommand updateAndCreateTopicList(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ long startTime = System.currentTimeMillis();
+
+ final CreateTopicListRequestBody requestBody = CreateTopicListRequestBody.decode(request.getBody(), CreateTopicListRequestBody.class);
+ List topicConfigList = requestBody.getTopicConfigList();
+
+ StringBuilder builder = new StringBuilder();
+ for (TopicConfig topicConfig : topicConfigList) {
+ builder.append(topicConfig.getTopicName()).append(";");
}
+ String topicNames = builder.toString();
+ LOGGER.info("AdminBrokerProcessor#updateAndCreateTopicList: topicNames: {}, called by {}", topicNames, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+
+ long executionTime;
try {
- this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
+ // Valid topics
+ for (TopicConfig topicConfig : topicConfigList) {
+ String topic = topicConfig.getTopicName();
+ TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
+ if (!result.isValid()) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark(result.getRemark());
+ return response;
+ }
+ if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
+ if (TopicValidator.isSystemTopic(topic)) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark("The topic[" + topic + "] is conflict with system topic.");
+ return response;
+ }
+ }
+ if (!brokerController.getBrokerConfig().isEnableMixedMessageType() && topicConfig.getAttributes() != null) {
+ // Get attribute by key with prefix sign
+ String msgTypeAttrKey = AttributeParser.ATTR_ADD_PLUS_SIGN + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName();
+ String msgTypeAttrValue = topicConfig.getAttributes().get(msgTypeAttrKey);
+ if (msgTypeAttrValue != null && msgTypeAttrValue.equals(TopicMessageType.MIXED.getValue())) {
+ response.setCode(ResponseCode.INVALID_PARAMETER);
+ response.setRemark("MIXED message type is not supported.");
+ return response;
+ }
+ }
+ if (topicConfig.equals(this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic))) {
+ LOGGER.info("Broker receive request to update or create topic={}, but topicConfig has no changes , so idempotent, caller address={}",
+ topic, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+ response.setCode(ResponseCode.SUCCESS);
+ return response;
+ }
+ }
+
+ this.brokerController.getTopicConfigManager().updateTopicConfigList(topicConfigList);
if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) {
- this.brokerController.registerSingleTopicAll(topicConfig);
+ for (TopicConfig topicConfig : topicConfigList) {
+ this.brokerController.registerSingleTopicAll(topicConfig);
+ }
} else {
- this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion());
+ this.brokerController.registerIncrementBrokerData(topicConfigList, this.brokerController.getTopicConfigManager().getDataVersion());
}
response.setCode(ResponseCode.SUCCESS);
} catch (Exception e) {
LOGGER.error("Update / create topic failed for [{}]", request, e);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(e.getMessage());
+ return response;
+ } finally {
+ executionTime = System.currentTimeMillis() - startTime;
+ InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+ InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+ Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+ .put(LABEL_INVOCATION_STATUS, status.getName())
+ .put(LABEL_IS_SYSTEM, TopicValidator.isSystemTopic(topicNames))
+ .build();
+ BrokerMetricsManager.topicCreateExecuteTime.record(executionTime, attributes);
}
-
+ LOGGER.info("executionTime of all topics:{} is {} ms", topicNames, executionTime);
return response;
}
@@ -532,21 +674,18 @@ private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerCo
TopicValidator.ValidateTopicResult result = TopicValidator.validateTopic(topic);
if (!result.isValid()) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(result.getRemark());
return response;
}
if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
if (TopicValidator.isSystemTopic(topic)) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The topic[" + topic + "] is conflict with system topic.");
return response;
}
}
- boolean force = false;
- if (requestHeader.getForce() != null && requestHeader.getForce()) {
- force = true;
- }
+ boolean force = requestHeader.getForce() != null && requestHeader.getForce();
TopicConfig topicConfig = new TopicConfig(topic);
topicConfig.setReadQueueNums(requestHeader.getReadQueueNums());
@@ -582,34 +721,41 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx,
String topic = requestHeader.getTopic();
if (UtilAll.isBlank(topic)) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The specified topic is blank.");
return response;
}
if (brokerController.getBrokerConfig().isValidateSystemTopicWhenUpdateTopic()) {
if (TopicValidator.isSystemTopic(topic)) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The topic[" + topic + "] is conflict with system topic.");
return response;
}
}
- final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
- // delete pop retry topics first
- try {
+ List topicsToClean = new ArrayList<>();
+ topicsToClean.add(topic);
+
+ if (brokerController.getBrokerConfig().isClearRetryTopicWhenDeleteTopic()) {
+ final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic);
for (String group : groups) {
final String popRetryTopicV2 = KeyBuilder.buildPopRetryTopic(topic, group, true);
if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV2) != null) {
- deleteTopicInBroker(popRetryTopicV2);
+ topicsToClean.add(popRetryTopicV2);
}
final String popRetryTopicV1 = KeyBuilder.buildPopRetryTopicV1(topic, group);
if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopicV1) != null) {
- deleteTopicInBroker(popRetryTopicV1);
+ topicsToClean.add(popRetryTopicV1);
}
}
- // delete topic
- deleteTopicInBroker(topic);
+ }
+
+ try {
+ for (String topicToClean : topicsToClean) {
+ // delete topic
+ deleteTopicInBroker(topicToClean);
+ }
} catch (Throwable t) {
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
@@ -626,26 +772,15 @@ private void deleteTopicInBroker(String topic) {
this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic));
}
- private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final CreateAccessConfigRequestHeader requestHeader =
- (CreateAccessConfigRequestHeader) request.decodeCommandCustomHeader(CreateAccessConfigRequestHeader.class);
-
- PlainAccessConfig accessConfig = new PlainAccessConfig();
- accessConfig.setAccessKey(requestHeader.getAccessKey());
- accessConfig.setSecretKey(requestHeader.getSecretKey());
- accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress());
- accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm());
- accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm());
- accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ","));
- accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ","));
- accessConfig.setAdmin(requestHeader.isAdmin());
try {
+ ensureAclEnabled();
+ final CreateAccessConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateAccessConfigRequestHeader.class);
AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
- if (accessValidator.updateAccessConfig(accessConfig)) {
+ if (accessValidator.updateAccessConfig(createAccessConfig(requestHeader))) {
response.setCode(ResponseCode.SUCCESS);
response.setOpaque(request.getOpaque());
response.markResponseType();
@@ -668,15 +803,28 @@ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerC
return null;
}
- private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
+ private PlainAccessConfig createAccessConfig(final CreateAccessConfigRequestHeader requestHeader) {
+ PlainAccessConfig accessConfig = new PlainAccessConfig();
+ accessConfig.setAccessKey(requestHeader.getAccessKey());
+ accessConfig.setSecretKey(requestHeader.getSecretKey());
+ accessConfig.setWhiteRemoteAddress(requestHeader.getWhiteRemoteAddress());
+ accessConfig.setDefaultTopicPerm(requestHeader.getDefaultTopicPerm());
+ accessConfig.setDefaultGroupPerm(requestHeader.getDefaultGroupPerm());
+ accessConfig.setTopicPerms(UtilAll.split(requestHeader.getTopicPerms(), ","));
+ accessConfig.setGroupPerms(UtilAll.split(requestHeader.getGroupPerms(), ","));
+ accessConfig.setAdmin(requestHeader.isAdmin());
+ return accessConfig;
+ }
+
+ private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final DeleteAccessConfigRequestHeader requestHeader =
- (DeleteAccessConfigRequestHeader) request.decodeCommandCustomHeader(DeleteAccessConfigRequestHeader.class);
LOGGER.info("DeleteAccessConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
try {
+ ensureAclEnabled();
+
+ final DeleteAccessConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(DeleteAccessConfigRequestHeader.class);
String accessKey = requestHeader.getAccessKey();
AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
if (accessValidator.deleteAccessConfig(accessKey)) {
@@ -703,15 +851,13 @@ private synchronized RemotingCommand deleteAccessConfig(ChannelHandlerContext ct
return null;
}
- private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandlerContext ctx,
- RemotingCommand request) throws RemotingCommandException {
-
+ private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandlerContext ctx, RemotingCommand request) {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader =
- (UpdateGlobalWhiteAddrsConfigRequestHeader) request.decodeCommandCustomHeader(UpdateGlobalWhiteAddrsConfigRequestHeader.class);
-
try {
+ ensureAclEnabled();
+
+ final UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateGlobalWhiteAddrsConfigRequestHeader.class);
AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
if (accessValidator.updateGlobalWhiteAddrsConfig(UtilAll.split(requestHeader.getGlobalWhiteAddrs(), ","),
requestHeader.getAclFileFullPath())) {
@@ -738,12 +884,12 @@ private synchronized RemotingCommand updateGlobalWhiteAddrsConfig(ChannelHandler
}
private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, RemotingCommand request) {
-
final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerAclConfigResponseHeader.class);
- final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader();
-
try {
+ ensureAclEnabled();
+
+ final GetBrokerAclConfigResponseHeader responseHeader = (GetBrokerAclConfigResponseHeader) response.readCustomHeader();
AccessValidator accessValidator = this.brokerController.getAccessValidatorMap().get(PlainAccessValidator.class);
responseHeader.setVersion(accessValidator.getAclConfigVersion());
@@ -756,9 +902,16 @@ private RemotingCommand getBrokerAclConfigVersion(ChannelHandlerContext ctx, Rem
return response;
} catch (Exception e) {
LOGGER.error("Failed to generate a proper getBrokerAclConfigVersion response", e);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
}
+ }
- return null;
+ private void ensureAclEnabled() {
+ if (!brokerController.getBrokerConfig().isAclEnable()) {
+ throw new AclException("The broker does not enable acl.");
+ }
}
private RemotingCommand getUnknownCmdResponse(ChannelHandlerContext ctx, RemotingCommand request) {
@@ -847,13 +1000,15 @@ private synchronized RemotingCommand updateColdDataFlowCtrGroupConfig(ChannelHan
Properties properties = MixAll.string2Properties(bodyStr);
if (properties != null) {
LOGGER.info("updateColdDataFlowCtrGroupConfig new config: {}, client: {}", properties, ctx.channel().remoteAddress());
- properties.entrySet().stream().forEach(i -> {
+ properties.forEach((key, value) -> {
try {
- String consumerGroup = String.valueOf(i.getKey());
- Long threshold = Long.valueOf(String.valueOf(i.getValue()));
- this.brokerController.getColdDataCgCtrService().addOrUpdateGroupConfig(consumerGroup, threshold);
+ String consumerGroup = String.valueOf(key);
+ Long threshold = Long.valueOf(String.valueOf(value));
+ this.brokerController.getColdDataCgCtrService()
+ .addOrUpdateGroupConfig(consumerGroup, threshold);
} catch (Exception e) {
- LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}", i.getKey(), i.getValue(), e);
+ LOGGER.error("updateColdDataFlowCtrGroupConfig properties on entry error, key: {}, val: {}",
+ key, value, e);
}
});
} else {
@@ -937,7 +1092,7 @@ private RemotingCommand setCommitLogReadaheadMode(ChannelHandlerContext ctx, Rem
}
int mode = Integer.parseInt(extFields.get(FIleReadaheadMode.READ_AHEAD_MODE));
if (mode != LibC.MADV_RANDOM && mode != LibC.MADV_NORMAL) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("set commitlog readahead mode param value error");
return response;
}
@@ -1163,8 +1318,7 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);
final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();
- final GetMaxOffsetRequestHeader requestHeader =
- (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
+ final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
@@ -1172,10 +1326,12 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx,
return rewriteResult;
}
- long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
-
- responseHeader.setOffset(offset);
-
+ try {
+ long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+ responseHeader.setOffset(offset);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
@@ -1306,7 +1462,8 @@ private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx,
return response;
}
- private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) {
+ private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request)
+ throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
HashMap runtimeInfo = this.prepareRuntimeInfo();
@@ -1450,6 +1607,7 @@ public void onException(Throwable e) {
private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
+ long startTime = System.currentTimeMillis();
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}",
@@ -1462,10 +1620,53 @@ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext c
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
+ long executionTime = System.currentTimeMillis() - startTime;
+ LOGGER.info("executionTime of create subscriptionGroup:{} is {} ms", config.getGroupName(), executionTime);
+ InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+ InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+ Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+ .put(LABEL_INVOCATION_STATUS, status.getName())
+ .build();
+ BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes);
+ return response;
+ }
+
+ private RemotingCommand updateAndCreateSubscriptionGroupList(ChannelHandlerContext ctx, RemotingCommand request) {
+ final long startTime = System.nanoTime();
+
+ final SubscriptionGroupList subscriptionGroupList = SubscriptionGroupList.decode(request.getBody(), SubscriptionGroupList.class);
+ final List groupConfigList = subscriptionGroupList.getGroupConfigList();
+
+ final StringBuilder builder = new StringBuilder();
+ for (SubscriptionGroupConfig config : groupConfigList) {
+ builder.append(config.getGroupName()).append(";");
+ }
+ final String groupNames = builder.toString();
+ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroupList: groupNames: {}, called by {}",
+ groupNames,
+ RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+
+ final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+ try {
+ this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfigList(groupConfigList);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } finally {
+ long executionTime = (System.nanoTime() - startTime) / 1000000L;
+ LOGGER.info("executionTime of create updateAndCreateSubscriptionGroupList: {} is {} ms", groupNames, executionTime);
+ InvocationStatus status = response.getCode() == ResponseCode.SUCCESS ?
+ InvocationStatus.SUCCESS : InvocationStatus.FAILURE;
+ Attributes attributes = BrokerMetricsManager.newAttributesBuilder()
+ .put(LABEL_INVOCATION_STATUS, status.getName())
+ .build();
+ BrokerMetricsManager.consumerGroupCreateExecuteTime.record(executionTime, attributes);
+ }
+
return response;
}
- private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) {
+ private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig)
+ throws ConsumeQueueException {
String topic = topicConfig.getTopicName();
for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) {
if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) {
@@ -1539,8 +1740,7 @@ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx,
private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final GetTopicStatsInfoRequestHeader requestHeader =
- (GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
+ final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
final String topic = requestHeader.getTopic();
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -1553,39 +1753,45 @@ private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,
TopicStatsTable topicStatsTable = new TopicStatsTable();
int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums());
- for (int i = 0; i < maxQueueNums; i++) {
- MessageQueue mq = new MessageQueue();
- mq.setTopic(topic);
- mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
- mq.setQueueId(i);
+ try {
+ for (int i = 0; i < maxQueueNums; i++) {
+ MessageQueue mq = new MessageQueue();
+ mq.setTopic(topic);
+ mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
+ mq.setQueueId(i);
- TopicOffset topicOffset = new TopicOffset();
- long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
- if (min < 0) {
- min = 0;
- }
+ TopicOffset topicOffset = new TopicOffset();
+ long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
+ if (min < 0) {
+ min = 0;
+ }
- long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
- if (max < 0) {
- max = 0;
- }
+ long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ if (max < 0) {
+ max = 0;
+ }
- long timestamp = 0;
- if (max > 0) {
- timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
- }
+ long timestamp = 0;
+ if (max > 0) {
+ timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
+ }
- topicOffset.setMinOffset(min);
- topicOffset.setMaxOffset(max);
- topicOffset.setLastUpdateTimestamp(timestamp);
+ topicOffset.setMinOffset(min);
+ topicOffset.setMaxOffset(max);
+ topicOffset.setLastUpdateTimestamp(timestamp);
+
+ topicStatsTable.getOffsetTable().put(mq, topicOffset);
+ }
- topicStatsTable.getOffsetTable().put(mq, topicOffset);
+ byte[] body = topicStatsTable.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } catch (ConsumeQueueException e) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
}
- byte[] body = topicStatsTable.encode();
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
@@ -1685,93 +1891,96 @@ private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx,
private RemotingCommand getConsumeStats(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- final GetConsumeStatsRequestHeader requestHeader =
- (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
-
- ConsumeStats consumeStats = new ConsumeStats();
-
- Set topics = new HashSet<>();
- if (UtilAll.isBlank(requestHeader.getTopic())) {
- topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
- } else {
- topics.add(requestHeader.getTopic());
- }
+ try {
+ final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
+ ConsumeStats consumeStats = new ConsumeStats();
- for (String topic : topics) {
- TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
- if (null == topicConfig) {
- LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
- continue;
+ Set topics = new HashSet<>();
+ if (UtilAll.isBlank(requestHeader.getTopic())) {
+ topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
+ } else {
+ topics.add(requestHeader.getTopic());
}
- TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
-
- {
- SubscriptionData findSubscriptionData =
- this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
-
- if (null == findSubscriptionData
- && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
- LOGGER.warn(
- "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, "
- + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup());
+ for (String topic : topics) {
+ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
+ if (null == topicConfig) {
+ LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
continue;
}
- }
- for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue();
- mq.setTopic(topic);
- mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
- mq.setQueueId(i);
+ TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
- OffsetWrapper offsetWrapper = new OffsetWrapper();
+ {
+ SubscriptionData findSubscriptionData =
+ this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
- long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
- if (brokerOffset < 0) {
- brokerOffset = 0;
+ if (null == findSubscriptionData
+ && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
+ LOGGER.warn(
+ "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, "
+ + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup());
+ continue;
+ }
}
- long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
- requestHeader.getConsumerGroup(), topic, i);
+ for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue();
+ mq.setTopic(topic);
+ mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
+ mq.setQueueId(i);
- // the consumerOffset cannot be zero for static topic because of the "double read check" strategy
- // just remain the logic for dynamic topic
- // maybe we should remove it in the future
- if (mappingDetail == null) {
- if (consumerOffset < 0) {
- consumerOffset = 0;
+ OffsetWrapper offsetWrapper = new OffsetWrapper();
+
+ long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ if (brokerOffset < 0) {
+ brokerOffset = 0;
}
- }
- long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
- requestHeader.getConsumerGroup(), topic, i);
+ long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
+ requestHeader.getConsumerGroup(), topic, i);
+
+ // the consumerOffset cannot be zero for static topic because of the "double read check" strategy
+ // just remain the logic for dynamic topic
+ // maybe we should remove it in the future
+ if (mappingDetail == null) {
+ if (consumerOffset < 0) {
+ consumerOffset = 0;
+ }
+ }
- offsetWrapper.setBrokerOffset(brokerOffset);
- offsetWrapper.setConsumerOffset(consumerOffset);
- offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
+ long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
+ requestHeader.getConsumerGroup(), topic, i);
- long timeOffset = consumerOffset - 1;
- if (timeOffset >= 0) {
- long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
- if (lastTimestamp > 0) {
- offsetWrapper.setLastTimestamp(lastTimestamp);
+ offsetWrapper.setBrokerOffset(brokerOffset);
+ offsetWrapper.setConsumerOffset(consumerOffset);
+ offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
+
+ long timeOffset = consumerOffset - 1;
+ if (timeOffset >= 0) {
+ long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
+ if (lastTimestamp > 0) {
+ offsetWrapper.setLastTimestamp(lastTimestamp);
+ }
}
+
+ consumeStats.getOffsetTable().put(mq, offsetWrapper);
}
- consumeStats.getOffsetTable().put(mq, offsetWrapper);
- }
+ double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
- double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
+ consumeTps += consumeStats.getConsumeTps();
+ consumeStats.setConsumeTps(consumeTps);
+ }
- consumeTps += consumeStats.getConsumeTps();
- consumeStats.setConsumeTps(consumeTps);
+ byte[] body = consumeStats.encode();
+ response.setBody(body);
+ response.setCode(ResponseCode.SUCCESS);
+ response.setRemark(null);
+ } catch (ConsumeQueueException e) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
}
-
- byte[] body = consumeStats.encode();
- response.setBody(body);
- response.setCode(ResponseCode.SUCCESS);
- response.setRemark(null);
return response;
}
@@ -1835,7 +2044,7 @@ private RemotingCommand getAllMessageRequestMode(ChannelHandlerContext ctx, Remo
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
String content = this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager().encode();
- if (content != null && content.length() > 0) {
+ if (content != null && !content.isEmpty()) {
try {
response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET));
} catch (UnsupportedEncodingException e) {
@@ -1886,7 +2095,7 @@ public RemotingCommand resetOffset(ChannelHandlerContext ctx,
requestHeader.getTimestamp(), requestHeader.isForce(), isC);
}
- private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) {
+ private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException {
if (timestamp < 0) {
return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
} else {
@@ -1897,13 +2106,13 @@ private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp)
/**
* Reset consumer offset.
*
- * @param topic Required, not null.
- * @param group Required, not null.
- * @param queueId if target queue ID is negative, all message queues will be reset; otherwise, only the target queue
- * would get reset.
+ * @param topic Required, not null.
+ * @param group Required, not null.
+ * @param queueId if target queue ID is negative, all message queues will be reset; otherwise, only the target queue
+ * would get reset.
* @param timestamp if timestamp is negative, offset would be reset to broker offset at the time being; otherwise,
- * binary search is performed to locate target offset.
- * @param offset Target offset to reset to if target queue ID is properly provided.
+ * binary search is performed to locate target offset.
+ * @param offset Target offset to reset to if target queue ID is properly provided.
* @return Affected queues and their new offset
*/
private RemotingCommand resetOffsetInner(String topic, String group, int queueId, long timestamp, Long offset) {
@@ -1918,7 +2127,7 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId
Map queueOffsetMap = new HashMap<>();
// Reset offset for all queues belonging to the specified topic
- TopicConfig topicConfig = brokerController.getTopicConfigManager().getTopicConfigTable().get(topic);
+ TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
if (null == topicConfig) {
response.setCode(ResponseCode.TOPIC_NOT_EXIST);
response.setRemark("Topic " + topic + " does not exist");
@@ -1933,25 +2142,31 @@ private RemotingCommand resetOffsetInner(String topic, String group, int queueId
return response;
}
- if (queueId >= 0) {
- if (null != offset && -1 != offset) {
- long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
- long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
- if (min >= 0 && offset < min || offset > max + 1) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
- response.setRemark(
- String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
- return response;
+ try {
+ if (queueId >= 0) {
+ if (null != offset && -1 != offset) {
+ long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+ long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+ if (min >= 0 && offset < min || offset > max + 1) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(
+ String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
+ return response;
+ }
+ } else {
+ offset = searchOffsetByTimestamp(topic, queueId, timestamp);
}
+ queueOffsetMap.put(queueId, offset);
} else {
- offset = searchOffsetByTimestamp(topic, queueId, timestamp);
- }
- queueOffsetMap.put(queueId, offset);
- } else {
- for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
- offset = searchOffsetByTimestamp(topic, index, timestamp);
- queueOffsetMap.put(index, offset);
+ for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
+ offset = searchOffsetByTimestamp(topic, index, timestamp);
+ queueOffsetMap.put(index, offset);
+ }
}
+ } catch (ConsumeQueueException e) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark(e.getMessage());
+ return response;
}
if (queueOffsetMap.isEmpty()) {
@@ -2058,8 +2273,7 @@ private RemotingCommand querySubscriptionByConsumer(ChannelHandlerContext ctx,
private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- QueryConsumeTimeSpanRequestHeader requestHeader =
- (QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
+ QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
final String topic = requestHeader.getTopic();
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -2081,7 +2295,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i);
timeSpan.setMinTimeStamp(minTime);
- long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ long max;
+ try {
+ max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
timeSpan.setMaxTimeStamp(maxTime);
@@ -2095,7 +2314,12 @@ private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
}
timeSpan.setConsumeTimeStamp(consumeTime);
- long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
+ long maxBrokerOffset;
+ try {
+ maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
if (consumerOffset < maxBrokerOffset) {
long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset);
timeSpan.setDelayTime(System.currentTimeMillis() - nextTime);
@@ -2330,8 +2554,7 @@ private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx,
private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
- GetConsumeStatsInBrokerHeader requestHeader =
- (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
+ GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
boolean isOrder = requestHeader.isOrder();
ConcurrentMap subscriptionGroups =
brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
@@ -2377,7 +2600,12 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx,
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
OffsetWrapper offsetWrapper = new OffsetWrapper();
- long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ long brokerOffset;
+ try {
+ brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset", e);
+ }
if (brokerOffset < 0) {
brokerOffset = 0;
}
@@ -2421,7 +2649,7 @@ private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx,
return response;
}
- private HashMap prepareRuntimeInfo() {
+ private HashMap prepareRuntimeInfo() throws RemotingCommandException {
HashMap runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();
for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {
@@ -2430,7 +2658,11 @@ private HashMap prepareRuntimeInfo() {
}
}
- this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
+ try {
+ this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
runtimeInfo.put("brokerActive", String.valueOf(this.brokerController.isSpecialServiceRunning()));
runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION));
@@ -2849,7 +3081,7 @@ private RemotingCommand createUser(ChannelHandlerContext ctx,
CreateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(CreateUserRequestHeader.class);
if (StringUtils.isEmpty(requestHeader.getUsername())) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The username is blank");
return response;
}
@@ -2881,7 +3113,7 @@ private RemotingCommand updateUser(ChannelHandlerContext ctx,
UpdateUserRequestHeader requestHeader = request.decodeCommandCustomHeader(UpdateUserRequestHeader.class);
if (StringUtils.isEmpty(requestHeader.getUsername())) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The username is blank");
return response;
}
@@ -2904,7 +3136,7 @@ private RemotingCommand updateUser(ChannelHandlerContext ctx,
if (old.getUserType() == UserType.SUPER && isNotSuperUserLogin(request)) {
throw new AuthenticationException("The super user can only be update by super user");
}
- return this.brokerController.getAuthenticationMetadataManager().updateUser(old);
+ return this.brokerController.getAuthenticationMetadataManager().updateUser(user);
}).thenAccept(nil -> response.setCode(ResponseCode.SUCCESS))
.exceptionally(ex -> {
LOGGER.error("update user {} error", requestHeader.getUsername(), ex);
@@ -2945,7 +3177,7 @@ private RemotingCommand getUser(ChannelHandlerContext ctx,
GetUserRequestHeader requestHeader = request.decodeCommandCustomHeader(GetUserRequestHeader.class);
if (StringUtils.isBlank(requestHeader.getUsername())) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("The username is blank");
return response;
}
@@ -3154,11 +3386,139 @@ private boolean validateSlave(RemotingCommand response) {
}
private boolean validateBlackListConfigExist(Properties properties) {
- for (String blackConfig:configBlackList) {
+ for (String blackConfig : configBlackList) {
if (properties.containsKey(blackConfig)) {
return true;
}
}
return false;
}
+
+ private CheckRocksdbCqWriteResult doCheckRocksdbCqWriteProgress(ChannelHandlerContext ctx,
+ RemotingCommand request) throws RemotingCommandException {
+ CheckRocksdbCqWriteProgressRequestHeader requestHeader = request.decodeCommandCustomHeader(CheckRocksdbCqWriteProgressRequestHeader.class);
+ String requestTopic = requestHeader.getTopic();
+ MessageStore messageStore = brokerController.getMessageStore();
+ DefaultMessageStore defaultMessageStore;
+ if (messageStore instanceof AbstractPluginMessageStore) {
+ defaultMessageStore = (DefaultMessageStore) ((AbstractPluginMessageStore) messageStore).getNext();
+ } else {
+ defaultMessageStore = (DefaultMessageStore) messageStore;
+ }
+ RocksDBMessageStore rocksDBMessageStore = defaultMessageStore.getRocksDBMessageStore();
+ CheckRocksdbCqWriteResult result = new CheckRocksdbCqWriteResult();
+
+ if (defaultMessageStore.getMessageStoreConfig().getStoreType().equals(StoreType.DEFAULT_ROCKSDB.getStoreType())) {
+ result.setCheckResult("storeType is DEFAULT_ROCKSDB, no need check");
+ result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue());
+ return result;
+ }
+
+ if (!defaultMessageStore.getMessageStoreConfig().isRocksdbCQDoubleWriteEnable()) {
+ result.setCheckResult("rocksdbCQWriteEnable is false, checkRocksdbCqWriteProgressCommand is invalid");
+ result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+ return result;
+ }
+
+ ConcurrentMap> cqTable = defaultMessageStore.getConsumeQueueTable();
+ StringBuilder diffResult = new StringBuilder();
+ try {
+ if (StringUtils.isNotBlank(requestTopic)) {
+ boolean checkResult = processConsumeQueuesForTopic(cqTable.get(requestTopic), requestTopic, rocksDBMessageStore, diffResult, true, requestHeader.getCheckStoreTime());
+ result.setCheckResult(diffResult.toString());
+ result.setCheckStatus(checkResult ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+ return result;
+ }
+ int successNum = 0;
+ int checkSize = 0;
+ for (Map.Entry> topicEntry : cqTable.entrySet()) {
+ boolean checkResult = processConsumeQueuesForTopic(topicEntry.getValue(), topicEntry.getKey(), rocksDBMessageStore, diffResult, false, requestHeader.getCheckStoreTime());
+ successNum += checkResult ? 1 : 0;
+ checkSize++;
+ }
+ // check all topic finish, all topic is ready, checkSize: 100, currentQueueNum: 110 -> ready (The currentQueueNum means when we do checking, new topics are added.)
+ // check all topic finish, success/all : 89/100, currentQueueNum: 110 -> not ready
+ boolean checkReady = successNum == checkSize;
+ String checkResultString = checkReady ? String.format("all topic is ready, checkSize: %s, currentQueueNum: %s", checkSize, cqTable.size()) :
+ String.format("success/all : %s/%s, currentQueueNum: %s", successNum, checkSize, cqTable.size());
+ diffResult.append("check all topic finish, ").append(checkResultString);
+ result.setCheckResult(diffResult.toString());
+ result.setCheckStatus(checkReady ? CheckRocksdbCqWriteResult.CheckStatus.CHECK_OK.getValue() : CheckRocksdbCqWriteResult.CheckStatus.CHECK_NOT_OK.getValue());
+ } catch (Exception e) {
+ LOGGER.error("CheckRocksdbCqWriteProgressCommand error", e);
+ result.setCheckResult(e.getMessage() + Arrays.toString(e.getStackTrace()));
+ result.setCheckStatus(CheckRocksdbCqWriteResult.CheckStatus.CHECK_ERROR.getValue());
+ }
+ return result;
+ }
+
+ private boolean processConsumeQueuesForTopic(ConcurrentMap queueMap, String topic,
+ RocksDBMessageStore rocksDBMessageStore, StringBuilder diffResult, boolean printDetail,
+ long checkpointByStoreTime) {
+ boolean processResult = true;
+ for (Map.Entry queueEntry : queueMap.entrySet()) {
+ Integer queueId = queueEntry.getKey();
+ ConsumeQueueInterface jsonCq = queueEntry.getValue();
+ ConsumeQueueInterface kvCq = rocksDBMessageStore.getConsumeQueue(topic, queueId);
+ if (printDetail) {
+ String format = String.format("[topic: %s, queue: %s] \n kvEarliest : %s | kvLatest : %s \n fileEarliest: %s | fileEarliest: %s ",
+ topic, queueId, kvCq.getEarliestUnit(), kvCq.getLatestUnit(), jsonCq.getEarliestUnit(), jsonCq.getLatestUnit());
+ diffResult.append(format).append("\n");
+ }
+
+ long minOffsetByTime = 0L;
+ try {
+ minOffsetByTime = rocksDBMessageStore.getConsumeQueueStore().getOffsetInQueueByTime(topic, queueId, checkpointByStoreTime, BoundaryType.UPPER);
+ } catch (Exception e) {
+ // ignore
+ }
+ long minOffsetInQueue = kvCq.getMinOffsetInQueue();
+ long checkFrom = Math.max(minOffsetInQueue, minOffsetByTime);
+ long checkTo = jsonCq.getMaxOffsetInQueue() - 1;
+ /*
+ checkTo(maxOffsetInQueue - 1)
+ v
+ fileCq +------------------------------------------------------+
+ kvCq +----------------------------------------------+
+ ^ ^
+ minOffsetInQueue minOffsetByTime
+ ^
+ checkFrom = max(minOffsetInQueue, minOffsetByTime)
+ */
+ // The latest message is earlier than the check time
+ Pair fileLatestCq = jsonCq.getCqUnitAndStoreTime(checkTo);
+ if (fileLatestCq != null) {
+ if (fileLatestCq.getObject2() < checkpointByStoreTime) {
+ continue;
+ }
+ }
+ for (long i = checkFrom; i <= checkTo; i++) {
+ Pair fileCqUnit = jsonCq.getCqUnitAndStoreTime(i);
+ Pair kvCqUnit = kvCq.getCqUnitAndStoreTime(i);
+ if (fileCqUnit == null || kvCqUnit == null || !checkCqUnitEqual(kvCqUnit.getObject1(), fileCqUnit.getObject1())) {
+ LOGGER.error(String.format("[topic: %s, queue: %s, offset: %s] \n file : %s \n kv : %s \n",
+ topic, queueId, i, kvCqUnit != null ? kvCqUnit.getObject1() : "null", fileCqUnit != null ? fileCqUnit.getObject1() : "null"));
+ processResult = false;
+ break;
+ }
+ }
+ }
+ return processResult;
+ }
+
+ private boolean checkCqUnitEqual(CqUnit cqUnit1, CqUnit cqUnit2) {
+ if (cqUnit1.getQueueOffset() != cqUnit2.getQueueOffset()) {
+ return false;
+ }
+ if (cqUnit1.getSize() != cqUnit2.getSize()) {
+ return false;
+ }
+ if (cqUnit1.getPos() != cqUnit2.getPos()) {
+ return false;
+ }
+ if (cqUnit1.getBatchNum() != cqUnit2.getBatchNum()) {
+ return false;
+ }
+ return cqUnit1.getTagsCode() == cqUnit2.getTagsCode();
+ }
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
index bdfffff096a..d29ff2a55b0 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
@@ -19,6 +19,9 @@
import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.nio.charset.StandardCharsets;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.metrics.PopMetricsManager;
import org.apache.rocketmq.common.PopAckConstants;
@@ -28,19 +31,19 @@
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
-import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.ResponseCode;
import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
-import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -67,6 +70,35 @@ public boolean rejectRequest() {
private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
boolean brokerAllowSuspend) throws RemotingCommandException {
+
+ CompletableFuture responseFuture = processRequestAsync(channel, request, brokerAllowSuspend);
+
+ if (brokerController.getBrokerConfig().isAppendCkAsync() && brokerController.getBrokerConfig().isAppendAckAsync()) {
+ responseFuture.thenAccept(response -> doResponse(channel, request, response)).exceptionally(throwable -> {
+ RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setOpaque(request.getOpaque());
+ doResponse(channel, request, response);
+ POP_LOGGER.error("append checkpoint or ack origin failed", throwable);
+ return null;
+ });
+ } else {
+ RemotingCommand response;
+ try {
+ response = responseFuture.get(3000, TimeUnit.MILLISECONDS);
+ } catch (Exception e) {
+ response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setOpaque(request.getOpaque());
+ POP_LOGGER.error("append checkpoint or ack origin failed", e);
+ }
+ return response;
+ }
+ return null;
+ }
+
+ public CompletableFuture processRequestAsync(final Channel channel, RemotingCommand request,
+ boolean brokerAllowSuspend) throws RemotingCommandException {
final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class);
RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
response.setCode(ResponseCode.SUCCESS);
@@ -77,7 +109,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
response.setCode(ResponseCode.TOPIC_NOT_EXIST);
response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
- return response;
+ return CompletableFuture.completedFuture(response);
}
if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
@@ -86,46 +118,40 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
POP_LOGGER.warn(errorInfo);
response.setCode(ResponseCode.MESSAGE_ILLEGAL);
response.setRemark(errorInfo);
- return response;
+ return CompletableFuture.completedFuture(response);
}
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
- long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+ long maxOffset;
+ try {
+ maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to get max consume offset", e);
+ }
if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
response.setCode(ResponseCode.NO_MESSAGE);
- return response;
+ return CompletableFuture.completedFuture(response);
}
String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());
if (ExtraInfoUtil.isOrder(extraInfo)) {
- return processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader);
+ return CompletableFuture.completedFuture(processChangeInvisibleTimeForOrder(requestHeader, extraInfo, response, responseHeader));
}
// add new ck
long now = System.currentTimeMillis();
- PutMessageResult ckResult = appendCheckPoint(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, ExtraInfoUtil.getBrokerName(extraInfo));
-
- if (ckResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
- && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
- && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
- && ckResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
- POP_LOGGER.error("change Invisible, put new ck error: {}", ckResult);
- response.setCode(ResponseCode.SYSTEM_ERROR);
- return response;
- }
- // ack old msg.
- try {
- ackOrigin(requestHeader, extraInfo);
- } catch (Throwable e) {
- POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage());
- // cancel new ck?
- }
-
- responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
- responseHeader.setPopTime(now);
- responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
- return response;
+ CompletableFuture futureResult = appendCheckPointThenAckOrigin(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, extraInfo);
+ return futureResult.thenCompose(result -> {
+ if (result) {
+ responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
+ responseHeader.setPopTime(now);
+ responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
+ } else {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ }
+ return CompletableFuture.completedFuture(response);
+ });
}
protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTimeRequestHeader requestHeader,
@@ -158,7 +184,8 @@ protected RemotingCommand processChangeInvisibleTimeForOrder(ChangeInvisibleTime
return response;
}
- private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo) {
+ private CompletableFuture ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader,
+ String[] extraInfo) {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
AckMsg ackMsg = new AckMsg();
@@ -176,11 +203,11 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str
this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {
- return;
+ return CompletableFuture.completedFuture(true);
}
msgInner.setTopic(reviveTopic);
- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
+ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(rqId);
msgInner.setTags(PopAckConstants.ACK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -189,18 +216,25 @@ private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, Str
msgInner.setDeliverTimeMs(ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo));
msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
- if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
- POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult);
- }
- PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
+ return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {
+ if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+ POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult);
+ }
+ PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
+ return CompletableFuture.completedFuture(true);
+ }).exceptionally(e -> {
+ POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage());
+ return false;
+ });
}
- private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader requestHeader, int reviveQid,
- int queueId, long offset, long popTime, String brokerName) {
+ private CompletableFuture appendCheckPointThenAckOrigin(
+ final ChangeInvisibleTimeRequestHeader requestHeader,
+ int reviveQid,
+ int queueId, long offset, long popTime, String[] extraInfo) {
// add check point msg to revive log
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(reviveTopic);
@@ -214,9 +248,9 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader
ck.setTopic(requestHeader.getTopic());
ck.setQueueId(queueId);
ck.addDiff(0);
- ck.setBrokerName(brokerName);
+ ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));
- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
+ msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(reviveQid);
msgInner.setTags(PopAckConstants.CK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -225,21 +259,36 @@ private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader
msgInner.setDeliverTimeMs(ck.getReviveTime() - PopAckConstants.ackTimeInterval);
msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck));
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
-
- if (brokerController.getBrokerConfig().isEnablePopLog()) {
- POP_LOGGER.info("change Invisible , appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,
- ck.getReviveTime(), putMessageResult);
- }
+ return this.brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenCompose(putMessageResult -> {
+ if (brokerController.getBrokerConfig().isEnablePopLog()) {
+ POP_LOGGER.info("change Invisible, appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,
+ ck.getReviveTime(), putMessageResult);
+ }
- if (putMessageResult != null) {
- PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus());
- if (putMessageResult.isOk()) {
- this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
- this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+ if (putMessageResult != null) {
+ PopMetricsManager.incPopReviveCkPutCount(ck, putMessageResult.getPutMessageStatus());
+ if (putMessageResult.isOk()) {
+ this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
+ this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+ }
}
- }
+ if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+ POP_LOGGER.error("change invisible, put new ck error: {}", putMessageResult);
+ return CompletableFuture.completedFuture(false);
+ } else {
+ return ackOrigin(requestHeader, extraInfo);
+ }
+ }).exceptionally(throwable -> {
+ POP_LOGGER.error("change invisible, put new ck error", throwable);
+ return null;
+ });
+ }
- return putMessageResult;
+ protected void doResponse(Channel channel, RemotingCommand request,
+ final RemotingCommand response) {
+ NettyRemotingAbstract.writeResponse(channel, request, response);
}
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
index 9b3ef603de7..dfa755d7c44 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
@@ -177,13 +177,13 @@ private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, Remoting
}
if (queueId == null) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("QueueId is null, topic is " + topic);
return response;
}
if (offset == null) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark("Offset is null, topic is " + topic);
return response;
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
index 6447500cbe6..6317d6ad7d2 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
@@ -18,6 +18,7 @@
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
+import java.util.Map;
import java.util.Objects;
import java.util.Random;
import org.apache.rocketmq.broker.BrokerController;
@@ -40,6 +41,7 @@
import org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader;
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class NotificationProcessor implements NettyRequestProcessor {
private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
@@ -58,8 +60,16 @@ public boolean rejectRequest() {
return false;
}
+ // When a new message is written to CommitLog, this method would be called.
+ // Suspended long polling will receive notification and be wakeup.
+ public void notifyMessageArriving(final String topic, final int queueId, long offset,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
+ this.popLongPollingService.notifyMessageArrivingWithRetryTopic(
+ topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
+ }
+
public void notifyMessageArriving(final String topic, final int queueId) {
- popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
+ this.popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
}
@Override
@@ -102,7 +112,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
POP_LOGGER.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
}
@@ -160,13 +170,15 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx,
return response;
}
- private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) {
+ private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader)
+ throws RemotingCommandException {
boolean hasMsg;
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName);
return hasMsgFromTopic(topicConfig, randomQ, requestHeader);
}
- private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) {
+ private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader)
+ throws RemotingCommandException {
boolean hasMsg;
if (topicConfig != null) {
for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
@@ -180,15 +192,19 @@ private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, Notificati
return false;
}
- private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) {
+ private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) throws RemotingCommandException {
if (Boolean.TRUE.equals(requestHeader.getOrder())) {
if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) {
return false;
}
}
long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId);
- long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
- return restNum > 0;
+ try {
+ long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
+ return restNum > 0;
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed tp get max offset in queue", e);
+ }
}
private long getPopOffset(String topic, String cid, int queueId) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
index 55552003d80..40117b74a54 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
@@ -51,6 +51,7 @@
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;
@@ -113,7 +114,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
LOG.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
}
@@ -229,13 +230,18 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,
PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel,
- long popTime) {
+ long popTime) throws RemotingCommandException {
String topic = isRetry ?
KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2())
: requestHeader.getTopic();
GetMessageResult getMessageTmpResult;
long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);
- restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+ try {
+ restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+ } catch (ConsumeQueueException e) {
+ LOG.error("Failed to get max offset in queue. topic={}, queue-id={}", topic, queueId, e);
+ throw new RemotingCommandException("Failed to get max offset in queue", e);
+ }
if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
return restNum;
}
@@ -258,8 +264,8 @@ private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult
BrokerMetricsManager.throughputOutTotal.add(getMessageResult.getBufferTotalSize(), attributes);
}
- for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) {
- getMessageResult.addMessage(mapedBuffer);
+ for (SelectMappedBufferResult mappedBuffer : getMessageTmpResult.getMessageMapedList()) {
+ getMessageResult.addMessage(mappedBuffer);
}
}
return restNum;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
index 65a4d7d7851..f7baac144e6 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java
@@ -89,7 +89,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
POP_LOGGER.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
index 8a85dd8fec8..9f10b483ddb 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java
@@ -216,7 +216,8 @@ private void scanGarbage() {
private void scan() {
long startTime = System.currentTimeMillis();
- int count = 0, countCk = 0;
+ AtomicInteger count = new AtomicInteger(0);
+ int countCk = 0;
Iterator> iterator = buffer.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry entry = iterator.next();
@@ -257,14 +258,14 @@ private void scan() {
} else if (pointWrapper.isJustOffset()) {
// just offset should be in store.
if (pointWrapper.getReviveQueueOffset() < 0) {
- putCkToStore(pointWrapper, false);
+ putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());
countCk++;
}
continue;
} else if (removeCk) {
// put buffer ak to store
if (pointWrapper.getReviveQueueOffset() < 0) {
- putCkToStore(pointWrapper, false);
+ putCkToStore(pointWrapper, this.brokerController.getBrokerConfig().isAppendCkAsync());
countCk++;
}
@@ -278,17 +279,12 @@ private void scan() {
for (byte i = 0; i < point.getNum(); i++) {
// reput buffer ak to store
if (DataConverter.getBit(pointWrapper.getBits().get(), i)
- && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
+ && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
indexList.add(i);
}
}
if (indexList.size() > 0) {
- if (putBatchAckToStore(pointWrapper, indexList)) {
- count += indexList.size();
- for (Byte i : indexList) {
- markBitCAS(pointWrapper.getToStoreBits(), i);
- }
- }
+ putBatchAckToStore(pointWrapper, indexList, count);
}
} finally {
indexList.clear();
@@ -297,11 +293,8 @@ private void scan() {
for (byte i = 0; i < point.getNum(); i++) {
// reput buffer ak to store
if (DataConverter.getBit(pointWrapper.getBits().get(), i)
- && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
- if (putAckToStore(pointWrapper, i)) {
- count++;
- markBitCAS(pointWrapper.getToStoreBits(), i);
- }
+ && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
+ putAckToStore(pointWrapper, i, count);
}
}
}
@@ -312,7 +305,6 @@ private void scan() {
}
iterator.remove();
counter.decrementAndGet();
- continue;
}
}
}
@@ -323,13 +315,13 @@ private void scan() {
if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) {
POP_LOGGER.warn("[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, " +
"PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
- eclipse, count, countCk, counter.get(), offsetBufferSize);
+ eclipse, count.get(), countCk, counter.get(), offsetBufferSize);
this.serving = false;
} else {
if (scanTimes % countOfSecond1 == 0) {
POP_LOGGER.info("[PopBuffer]scan, PopBufferEclipse={}, " +
"PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
- eclipse, count, countCk, counter.get(), offsetBufferSize);
+ eclipse, count.get(), countCk, counter.get(), offsetBufferSize);
}
}
PopMetricsManager.recordPopBufferScanTimeConsume(eclipse);
@@ -429,7 +421,8 @@ private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) {
* @param nextBeginOffset
* @return
*/
- public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) {
+ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset,
+ long nextBeginOffset) {
PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true);
if (this.buffer.containsKey(pointWrapper.getMergeKey())) {
@@ -439,7 +432,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi
return false;
}
- this.putCkToStore(pointWrapper, !checkQueueOk(pointWrapper));
+ this.putCkToStore(pointWrapper, checkQueueOk(pointWrapper));
putOffsetQueue(pointWrapper);
this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);
@@ -447,7 +440,7 @@ public boolean addCkJustOffset(PopCheckPoint point, int reviveQueueId, long revi
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("[PopBuffer]add ck just offset, {}", pointWrapper);
}
- return true;
+ return true;
}
public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime,
@@ -597,13 +590,32 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean
if (pointWrapper.getReviveQueueOffset() >= 0) {
return;
}
+
MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId());
- PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+
+ // Indicates that ck message is storing
+ pointWrapper.setReviveQueueOffset(Long.MAX_VALUE);
+ if (brokerController.getBrokerConfig().isAppendCkAsync() && runInCurrent) {
+ brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+ handleCkMessagePutResult(putMessageResult, pointWrapper);
+ }).exceptionally(throwable -> {
+ POP_LOGGER.error("[PopBuffer]put ck to store fail: {}", pointWrapper, throwable);
+ pointWrapper.setReviveQueueOffset(-1);
+ return null;
+ });
+ } else {
+ PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ handleCkMessagePutResult(putMessageResult, pointWrapper);
+ }
+ }
+
+ private void handleCkMessagePutResult(PutMessageResult putMessageResult, final PopCheckPointWrapper pointWrapper) {
PopMetricsManager.incPopReviveCkPutCount(pointWrapper.getCk(), putMessageResult.getPutMessageStatus());
if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+ pointWrapper.setReviveQueueOffset(-1);
POP_LOGGER.error("[PopBuffer]put ck to store fail: {}, {}", pointWrapper, putMessageResult);
return;
}
@@ -621,7 +633,7 @@ private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean
}
}
- private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex) {
+ private void putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex, AtomicInteger count) {
PopCheckPoint point = pointWrapper.getCk();
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
final AckMsg ackMsg = new AckMsg();
@@ -632,6 +644,7 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI
ackMsg.setTopic(point.getTopic());
ackMsg.setQueueId(point.getQueueId());
ackMsg.setPopTime(point.getPopTime());
+ ackMsg.setBrokerName(point.getBrokerName());
msgInner.setTopic(popMessageProcessor.reviveTopic);
msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
msgInner.setQueueId(pointWrapper.getReviveQueueId());
@@ -643,23 +656,39 @@ private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgI
msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+
+ if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+ brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+ handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);
+ }).exceptionally(throwable -> {
+ POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}", pointWrapper, ackMsg, throwable);
+ return null;
+ });
+ } else {
+ PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ handleAckPutMessageResult(ackMsg, putMessageResult, pointWrapper, count, msgIndex);
+ }
+ }
+
+ private void handleAckPutMessageResult(AckMsg ackMsg, PutMessageResult putMessageResult,
+ PopCheckPointWrapper pointWrapper, AtomicInteger count, byte msgIndex) {
PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus());
if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
&& putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
- return false;
+ return;
}
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("[PopBuffer]put ack to store ok: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
}
-
- return true;
+ count.incrementAndGet();
+ markBitCAS(pointWrapper.getToStoreBits(), msgIndex);
}
- private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List msgIndexList) {
+ private void putBatchAckToStore(final PopCheckPointWrapper pointWrapper, final List msgIndexList,
+ AtomicInteger count) {
PopCheckPoint point = pointWrapper.getCk();
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
final BatchAckMsg batchAckMsg = new BatchAckMsg();
@@ -683,19 +712,36 @@ private boolean putBatchAckToStore(final PopCheckPointWrapper pointWrapper, fina
msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genBatchAckUniqueId(batchAckMsg));
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ if (brokerController.getBrokerConfig().isAppendAckAsync()) {
+ brokerController.getEscapeBridge().asyncPutMessageToSpecificQueue(msgInner).thenAccept(putMessageResult -> {
+ handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);
+ }).exceptionally(throwable -> {
+ POP_LOGGER.error("[PopBuffer]put batchAckMsg to store fail: {}, {}", pointWrapper, batchAckMsg, throwable);
+ return null;
+ });
+ } else {
+ PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+ handleBatchAckPutMessageResult(batchAckMsg, putMessageResult, pointWrapper, count, msgIndexList);
+ }
+ }
+
+ private void handleBatchAckPutMessageResult(BatchAckMsg batchAckMsg, PutMessageResult putMessageResult,
+ PopCheckPointWrapper pointWrapper, AtomicInteger count, List msgIndexList) {
if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
- && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+ && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
POP_LOGGER.error("[PopBuffer]put batch ack to store fail: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult);
- return false;
+ return;
}
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("[PopBuffer]put batch ack to store ok: {}, {}, {}", pointWrapper, batchAckMsg, putMessageResult);
}
- return true;
+ count.addAndGet(msgIndexList.size());
+ for (Byte i : msgIndexList) {
+ markBitCAS(pointWrapper.getToStoreBits(), i);
+ }
}
private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) {
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
index 93c04a1b8de..05efc14b7b4 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
@@ -24,8 +24,10 @@
import io.netty.channel.FileRegion;
import io.opentelemetry.api.common.Attributes;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Random;
@@ -61,9 +63,9 @@
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.topic.TopicValidator;
-import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.CommandCallback;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager;
@@ -82,6 +84,7 @@
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -95,8 +98,10 @@
import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_RESULT;
public class PopMessageProcessor implements NettyRequestProcessor {
+
private static final Logger POP_LOGGER =
LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+
private final BrokerController brokerController;
private final Random random = new Random(System.currentTimeMillis());
String reviveTopic;
@@ -166,16 +171,25 @@ public ConcurrentLinkedHashMap> getPol
return popLongPollingService.getPollingMap();
}
- public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) {
+ public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException {
+ this.notifyLongPollingRequestIfNeed(
+ topic, group, queueId, null, 0L, null, null);
+ }
+
+ public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap,
+ Map properties) throws ConsumeQueueException {
long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId);
long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
long offset = Math.max(popBufferOffset, consumerOffset);
if (maxOffset > offset) {
- boolean notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, -1);
+ boolean notifySuccess = popLongPollingService.notifyMessageArriving(
+ topic, -1, group, tagsCode, msgStoreTime, filterBitMap, properties);
if (!notifySuccess) {
// notify pop queue
- notifySuccess = popLongPollingService.notifyMessageArriving(topic, group, queueId);
+ notifySuccess = popLongPollingService.notifyMessageArriving(
+ topic, queueId, group, tagsCode, msgStoreTime, filterBitMap, properties);
}
this.brokerController.getNotificationProcessor().notifyMessageArriving(topic, queueId);
if (this.brokerController.getBrokerConfig().isEnablePopLog()) {
@@ -185,12 +199,15 @@ public void notifyLongPollingRequestIfNeed(String topic, String group, int queue
}
}
- public void notifyMessageArriving(final String topic, final int queueId) {
- popLongPollingService.notifyMessageArrivingWithRetryTopic(topic, queueId);
+ public void notifyMessageArriving(final String topic, final int queueId, long offset,
+ Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) {
+ popLongPollingService.notifyMessageArrivingWithRetryTopic(
+ topic, queueId, offset, tagsCode, msgStoreTime, filterBitMap, properties);
}
- public boolean notifyMessageArriving(final String topic, final String cid, final int queueId) {
- return popLongPollingService.notifyMessageArriving(topic, cid, queueId);
+ public void notifyMessageArriving(final String topic, final int queueId, final String cid) {
+ popLongPollingService.notifyMessageArriving(
+ topic, queueId, cid, false, null, 0L, null, null);
}
@Override
@@ -205,8 +222,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
- final PopMessageRequestHeader requestHeader =
- (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
+ final PopMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
StringBuilder startOffsetInfo = new StringBuilder(64);
StringBuilder msgOffsetInfo = new StringBuilder(64);
StringBuilder orderCountInfo = null;
@@ -236,7 +252,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
return response;
}
if (requestHeader.getMaxMsgNums() > 32) {
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(String.format("the broker[%s] pop message's num is greater than 32",
this.brokerController.getBrokerConfig().getBrokerIP1()));
return response;
@@ -272,7 +288,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(),
channel.remoteAddress());
POP_LOGGER.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
}
@@ -292,10 +308,11 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
}
BrokerConfig brokerConfig = brokerController.getBrokerConfig();
+ SubscriptionData subscriptionData = null;
ExpressionMessageFilter messageFilter = null;
- if (requestHeader.getExp() != null && requestHeader.getExp().length() > 0) {
+ if (requestHeader.getExp() != null && !requestHeader.getExp().isEmpty()) {
try {
- SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());
+ subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());
brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
requestHeader.getTopic(), subscriptionData);
@@ -329,7 +346,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
}
} else {
try {
- SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG);
+ subscriptionData = FilterAPI.build(requestHeader.getTopic(), "*", ExpressionType.TAG);
brokerController.getConsumerManager().compensateSubscribeData(requestHeader.getConsumerGroup(),
requestHeader.getTopic(), subscriptionData);
@@ -360,7 +377,7 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
// considered the same type because they share the same retry flag in previous fields.
// Therefore, needRetryV1 is designed as a subset of needRetry, and within a single request,
// only one type of retry topic is able to call popMsgFromQueue.
- boolean needRetry = randomQ % 5 == 0;
+ boolean needRetry = randomQ < brokerConfig.getPopFromRetryProbability();
boolean needRetryV1 = false;
if (brokerConfig.isEnableRetryTopicV2() && brokerConfig.isRetrieveMessageFromPopRetryTopicV1()) {
needRetryV1 = randomQ % 2 == 0;
@@ -403,18 +420,35 @@ public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingC
}
final RemotingCommand finalResponse = response;
+ SubscriptionData finalSubscriptionData = subscriptionData;
getMessageFuture.thenApply(restNum -> {
+ try {
+ if (request.getCallbackList() != null) {
+ request.getCallbackList().forEach(CommandCallback::accept);
+ request.getCallbackList().clear();
+ }
+ } catch (Throwable t) {
+ POP_LOGGER.error("PopProcessor execute callback error", t);
+ }
+
if (!getMessageResult.getMessageBufferList().isEmpty()) {
finalResponse.setCode(ResponseCode.SUCCESS);
getMessageResult.setStatus(GetMessageStatus.FOUND);
if (restNum > 0) {
// all queue pop can not notify specified queue pop, and vice versa
- popLongPollingService.notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(),
- requestHeader.getQueueId());
+ popLongPollingService.notifyMessageArriving(
+ requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+ null, 0L, null, null);
}
} else {
- PollingResult pollingResult = popLongPollingService.polling(ctx, request, new PollingHeader(requestHeader));
+ PollingResult pollingResult = popLongPollingService.polling(
+ ctx, request, new PollingHeader(requestHeader), finalSubscriptionData, finalMessageFilter);
if (PollingResult.POLLING_SUC == pollingResult) {
+ if (restNum > 0) {
+ popLongPollingService.notifyMessageArriving(
+ requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getConsumerGroup(),
+ null, 0L, null, null);
+ }
return null;
} else if (PollingResult.POLLING_FULL == pollingResult) {
finalResponse.setCode(ResponseCode.POLLING_FULL);
@@ -510,38 +544,61 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId,
String lockKey =
topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId;
boolean isOrder = requestHeader.isOrder();
- long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
- false, lockKey, false);
+ long offset;
+ try {
+ offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
+ false, lockKey, false);
+ } catch (ConsumeQueueException e) {
+ CompletableFuture failure = new CompletableFuture<>();
+ failure.completeExceptionally(e);
+ return failure;
+ }
+
CompletableFuture future = new CompletableFuture<>();
if (!queueLockManager.tryLock(lockKey)) {
- restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
- future.complete(restNum);
+ try {
+ restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+ future.complete(restNum);
+ } catch (ConsumeQueueException e) {
+ future.completeExceptionally(e);
+ }
return future;
}
+ future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));
if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) {
- POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId);
- restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
- future.complete(restNum);
+ POP_LOGGER.warn("Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}",
+ topic, requestHeader.getConsumerGroup(), queueId);
+ try {
+ restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+ future.complete(restNum);
+ } catch (ConsumeQueueException e) {
+ future.completeExceptionally(e);
+ }
return future;
}
try {
- future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));
offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
true, lockKey, true);
- if (isOrder && brokerController.getConsumerOrderInfoManager().checkBlock(attemptId, topic,
- requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {
- future.complete(this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum);
- return future;
- }
+ // Current requests would calculate the total number of messages
+ // waiting to be filtered for new message arrival notifications in
+ // the long-polling service, need disregarding the backlog in order
+ // consumption scenario. If rest message num including the blocked
+ // queue accumulation would lead to frequent unnecessary wake-ups
+ // of long-polling requests, resulting unnecessary CPU usage.
+ // When client ack message, long-polling request would be notifications
+ // by AckMessageProcessor.ackOrderly() and message will not be delayed.
if (isOrder) {
+ if (brokerController.getConsumerOrderInfoManager().checkBlock(
+ attemptId, topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {
+ // should not add accumulation(max offset - consumer offset) here
+ future.complete(restNum);
+ return future;
+ }
this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNum(
- topic,
- requestHeader.getConsumerGroup(),
- queueId
- );
+ topic, requestHeader.getConsumerGroup(), queueId);
}
if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
@@ -583,7 +640,11 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId,
return CompletableFuture.completedFuture(result);
}).thenApply(result -> {
if (result == null) {
- atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
+ try {
+ atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
+ } catch (ConsumeQueueException e) {
+ POP_LOGGER.error("Failed to get max offset in queue", e);
+ }
return atomicRestNum.get();
}
if (!result.getMessageMapedList().isEmpty()) {
@@ -622,10 +683,13 @@ private CompletableFuture popMsgFromQueue(String topic, String attemptId,
|| GetMessageStatus.MESSAGE_WAS_REMOVING.equals(result.getStatus())
|| GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(result.getStatus()))
&& result.getNextBeginOffset() > -1) {
- popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset,
- requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());
-// this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
-// queueId, getMessageTmpResult.getNextBeginOffset());
+ if (isOrder) {
+ this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
+ queueId, result.getNextBeginOffset());
+ } else {
+ popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, finalOffset,
+ requestHeader.getInvisibleTime(), popTime, reviveQid, result.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());
+ }
}
atomicRestNum.set(result.getMaxOffset() - result.getNextBeginOffset() + atomicRestNum.get());
@@ -680,7 +744,7 @@ private boolean isPopShouldStop(String topic, String group, int queueId) {
}
private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey,
- boolean checkResetOffset) {
+ boolean checkResetOffset) throws ConsumeQueueException {
long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
if (offset < 0) {
@@ -702,10 +766,11 @@ private long getPopOffset(String topic, String group, int queueId, int initMode,
}
}
- private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) {
+ private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init)
+ throws ConsumeQueueException {
long offset;
- if (ConsumeInitMode.MIN == initMode) {
- return this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+ if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+ offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
} else {
if (this.brokerController.getBrokerConfig().isInitPopOffsetByCheckMsgInMem() &&
this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId) <= 0 &&
@@ -719,10 +784,10 @@ private long getInitOffset(String topic, String group, int queueId, int initMode
offset = 0;
}
}
- if (init) {
- this.brokerController.getConsumerOffsetManager().commitOffset(
+ }
+ if (init) { // whichever initMode
+ this.brokerController.getConsumerOffsetManager().commitOffset(
"getPopOffset", group, topic, queueId, offset);
- }
}
return offset;
}
@@ -731,7 +796,7 @@ public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(reviveTopic);
- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
+ msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(reviveQid);
msgInner.setTags(PopAckConstants.CK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -762,6 +827,9 @@ private boolean appendCheckPoint(final PopMessageRequestHeader requestHeader,
ck.addDiff((int) (msgQueueOffset - offset));
}
+ this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
+ this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+
final boolean addBufferSuc = this.popBufferMergeService.addCk(
ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()
);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
index 4fab3d500ba..e1ead86169b 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
@@ -19,6 +19,7 @@
import com.alibaba.fastjson.JSON;
import io.opentelemetry.api.common.Attributes;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -27,6 +28,8 @@
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Triple;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.metrics.BrokerMetricsManager;
import org.apache.rocketmq.broker.metrics.PopMetricsManager;
@@ -34,6 +37,7 @@
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.KeyBuilder;
import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.PopAckConstants;
import org.apache.rocketmq.common.ServiceThread;
@@ -51,8 +55,8 @@
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -63,6 +67,7 @@
public class PopReviveService extends ServiceThread {
private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+ private final int[] ckRewriteIntervalsInSeconds = new int[] { 10, 20, 30, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 1200, 1800, 3600, 7200 };
private int queueId;
private BrokerController brokerController;
@@ -125,7 +130,7 @@ private boolean reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt)
msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime()));
}
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
- addRetryTopicIfNoExit(msgInner.getTopic(), popCheckPoint.getCId());
+ addRetryTopicIfNotExist(msgInner.getTopic(), popCheckPoint.getCId());
PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
PopMetricsManager.incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus());
if (brokerController.getBrokerConfig().isEnablePopLog()) {
@@ -153,7 +158,7 @@ private void initPopRetryOffset(String topic, String consumerGroup) {
}
}
- private void addRetryTopicIfNoExit(String topic, String consumerGroup) {
+ public void addRetryTopicIfNotExist(String topic, String consumerGroup) {
if (brokerController != null) {
TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
if (topicConfig != null) {
@@ -196,9 +201,9 @@ private boolean reachTail(PullResult pullResult, long offset) {
|| pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset();
}
- private CompletableFuture> getBizMessage(String topic, long offset, int queueId,
- String brokerName) {
- return this.brokerController.getEscapeBridge().getMessageAsync(topic, offset, queueId, brokerName, false);
+ // Triple
+ public CompletableFuture> getBizMessage(PopCheckPoint popCheckPoint, long offset) {
+ return this.brokerController.getEscapeBridge().getMessageAsync(popCheckPoint.getTopic(), offset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName(), false);
}
public PullResult getMessage(String group, String topic, int queueId, long offset, int nums,
@@ -258,10 +263,14 @@ public PullResult getMessage(String group, String topic, int queueId, long offse
getMessageResult.getMaxOffset(), foundList);
} else {
- long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
- if (maxQueueOffset > offset) {
- POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
- topic, group, offset, maxQueueOffset);
+ try {
+ long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
+ if (maxQueueOffset > offset) {
+ POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
+ topic, group, offset, maxQueueOffset);
+ }
+ } catch (ConsumeQueueException e) {
+ POP_LOGGER.error("Failed to get max offset in queue", e);
}
return null;
}
@@ -322,7 +331,7 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
if (endTime != 0 && System.currentTimeMillis() - endTime > 3 * PopAckConstants.SECOND && timerDelay <= 0 && commitLogDelay <= 0) {
endTime = System.currentTimeMillis();
}
- POP_LOGGER.info("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ",
+ POP_LOGGER.debug("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}, timerDelay={}, commitLogDelay={} ",
queueId, offset, old, endTime, timerDelay, commitLogDelay);
if (endTime - firstRt > PopAckConstants.ackTimeInterval + PopAckConstants.SECOND) {
break;
@@ -355,20 +364,22 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
if (point.getTopic() == null || point.getCId() == null) {
continue;
}
- map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(), point);
+ map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName(), point);
PopMetricsManager.incPopReviveCkGetCount(point, queueId);
point.setReviveOffset(messageExt.getQueueOffset());
if (firstRt == 0) {
firstRt = point.getReviveTime();
}
} else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {
- String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
+ String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
}
AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class);
PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId);
- String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime();
+ String brokerName = StringUtils.isNotBlank(ackMsg.getBrokerName()) ?
+ ackMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();
+ String mergeKey = ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + brokerName;
PopCheckPoint point = map.get(mergeKey);
if (point == null) {
if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {
@@ -386,14 +397,16 @@ protected void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
}
}
} else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) {
- String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
+ String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
}
BatchAckMsg bAckMsg = JSON.parseObject(raw, BatchAckMsg.class);
PopMetricsManager.incPopReviveAckGetCount(bAckMsg, queueId);
- String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime();
+ String brokerName = StringUtils.isNotBlank(bAckMsg.getBrokerName()) ?
+ bAckMsg.getBrokerName() : brokerController.getBrokerConfig().getBrokerName();
+ String mergeKey = bAckMsg.getTopic() + bAckMsg.getConsumerGroup() + bAckMsg.getQueueId() + bAckMsg.getStartOffset() + bAckMsg.getPopTime() + brokerName;
PopCheckPoint point = map.get(mergeKey);
if (point == null) {
if (!brokerController.getBrokerConfig().isEnableSkipLongAwaitingAck()) {
@@ -491,6 +504,8 @@ protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwabl
PopCheckPoint oldCK = inflightReviveRequestMap.firstKey();
rePutCK(oldCK, pair);
inflightReviveRequestMap.remove(oldCK);
+ POP_LOGGER.warn("stay too long, remove from reviveRequestMap, {}, {}, {}, {}", popCheckPoint.getTopic(),
+ popCheckPoint.getBrokerName(), popCheckPoint.getQueueId(), popCheckPoint.getStartOffset());
}
}
@@ -523,23 +538,13 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) {
// retry msg
long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j);
- CompletableFuture> future = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName())
- .thenApply(resultPair -> {
- GetMessageStatus getMessageStatus = resultPair.getObject1();
- MessageExt message = resultPair.getObject2();
+ CompletableFuture> future = getBizMessage(popCheckPoint, msgOffset)
+ .thenApply(rst -> {
+ MessageExt message = rst.getLeft();
if (message == null) {
- POP_LOGGER.warn("reviveQueueId={}, can not get biz msg topic is {}, offset is {}, then continue",
- queueId, popCheckPoint.getTopic(), msgOffset);
- switch (getMessageStatus) {
- case MESSAGE_WAS_REMOVING:
- case OFFSET_TOO_SMALL:
- case NO_MATCHED_LOGIC_QUEUE:
- case NO_MESSAGE_IN_QUEUE:
- return new Pair<>(msgOffset, true);
- default:
- return new Pair<>(msgOffset, false);
-
- }
+ POP_LOGGER.info("reviveQueueId={}, can not get biz msg, topic:{}, qid:{}, offset:{}, brokerName:{}, info:{}, retry:{}, then continue",
+ queueId, popCheckPoint.getTopic(), popCheckPoint.getQueueId(), msgOffset, popCheckPoint.getBrokerName(), UtilAll.frontStringAtLeast(rst.getMiddle(), 60), rst.getRight());
+ return new Pair<>(msgOffset, !rst.getRight()); // Pair.object2 means OK or not, Triple.right value means needRetry
}
boolean result = reviveRetry(popCheckPoint, message);
return new Pair<>(msgOffset, result);
@@ -572,6 +577,13 @@ private void reviveMsgFromCk(PopCheckPoint popCheckPoint) {
}
private void rePutCK(PopCheckPoint oldCK, Pair pair) {
+ int rePutTimes = oldCK.parseRePutTimes();
+ if (rePutTimes >= ckRewriteIntervalsInSeconds.length && brokerController.getBrokerConfig().isSkipWhenCKRePutReachMaxTimes()) {
+ POP_LOGGER.warn("rePut CK reach max times, drop it. {}, {}, {}, {}-{}, {}, {}, {}", oldCK.getTopic(), oldCK.getCId(),
+ oldCK.getBrokerName(), oldCK.getQueueId(), pair.getObject1(), oldCK.getPopTime(), oldCK.getInvisibleTime(), rePutTimes);
+ return;
+ }
+
PopCheckPoint newCk = new PopCheckPoint();
newCk.setBitMap(0);
newCk.setNum((byte) 1);
@@ -583,11 +595,17 @@ private void rePutCK(PopCheckPoint oldCK, Pair pair) {
newCk.setQueueId(oldCK.getQueueId());
newCk.setBrokerName(oldCK.getBrokerName());
newCk.addDiff(0);
+ newCk.setRePutTimes(String.valueOf(rePutTimes + 1)); // always increment even if removed from reviveRequestMap
+ if (oldCK.getReviveTime() <= System.currentTimeMillis()) {
+ // never expect an ACK matched in the future, we just use it to rewrite CK and try to revive retry message next time
+ int intervalIndex = rePutTimes >= ckRewriteIntervalsInSeconds.length ? ckRewriteIntervalsInSeconds.length - 1 : rePutTimes;
+ newCk.setInvisibleTime(oldCK.getInvisibleTime() + ckRewriteIntervalsInSeconds[intervalIndex] * 1000);
+ }
MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId);
brokerController.getMessageStore().putMessage(ckMsg);
}
- public long getReviveBehindMillis() {
+ public long getReviveBehindMillis() throws ConsumeQueueException {
if (currentReviveMessageTimestamp <= 0) {
return 0;
}
@@ -598,7 +616,7 @@ public long getReviveBehindMillis() {
return 0;
}
- public long getReviveBehindMessages() {
+ public long getReviveBehindMessages() throws ConsumeQueueException {
if (currentReviveMessageTimestamp <= 0) {
return 0;
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
index d53454f215d..5b11bc2fef4 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
@@ -73,6 +73,7 @@
import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
@@ -298,7 +299,8 @@ public boolean rejectRequest() {
return false;
}
- private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, boolean brokerAllowFlowCtrSuspend)
+ private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend,
+ boolean brokerAllowFlowCtrSuspend)
throws RemotingCommandException {
final long beginTimeMills = this.brokerController.getMessageStore().now();
RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
@@ -369,7 +371,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
LOGGER.warn(errorInfo);
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.INVALID_PARAMETER);
response.setRemark(errorInfo);
return response;
}
@@ -489,7 +491,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
final MessageStore messageStore = brokerController.getMessageStore();
if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {
- DefaultMessageStore defaultMessageStore = (DefaultMessageStore)this.brokerController.getMessageStore();
+ DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup());
if (cgNeedColdDataFlowCtr) {
boolean isMsgLogicCold = defaultMessageStore.getCommitLog()
@@ -526,7 +528,11 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re
getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET);
getMessageResult.setNextBeginOffset(resetOffset);
getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId));
- getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
+ try {
+ getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed tp get max offset in queue", e);
+ }
getMessageResult.setSuggestPullingFromSlave(false);
} else {
long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel);
@@ -589,12 +595,13 @@ public boolean hasConsumeMessageHook() {
/**
* Composes the header of the response message to be sent back to the client
- * @param requestHeader - the header of the request message
- * @param getMessageResult - the result of the GetMessage request
- * @param topicSysFlag - the system flag of the topic
+ *
+ * @param requestHeader - the header of the request message
+ * @param getMessageResult - the result of the GetMessage request
+ * @param topicSysFlag - the system flag of the topic
* @param subscriptionGroupConfig - configuration of the subscription group
- * @param response - the response message to be sent back to the client
- * @param clientAddress - the address of the client
+ * @param response - the response message to be sent back to the client
+ * @param clientAddress - the address of the client
*/
protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult,
int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response,
@@ -799,7 +806,7 @@ public void executeRequestWhenWakeup(final Channel channel, final RemotingComman
}
}
} catch (RemotingCommandException e1) {
- LOGGER.error("excuteRequestWhenWakeup run", e1);
+ LOGGER.error("executeRequestWhenWakeup run", e1);
}
};
this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, channel, request));
@@ -855,7 +862,7 @@ protected void updateBroadcastPulledOffset(String topic, String group, int queue
* When pull request is not broadcast or not return -1
*/
protected long queryBroadcastPullInitOffset(String topic, String group, int queueId,
- PullMessageRequestHeader requestHeader, Channel channel) {
+ PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException {
if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) {
return -1L;
@@ -877,8 +884,12 @@ protected long queryBroadcastPullInitOffset(String topic, String group, int queu
clientId = clientChannelInfo.getClientId();
}
- return this.brokerController.getBroadcastOffsetManager()
- .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
+ try {
+ return this.brokerController.getBroadcastOffsetManager()
+ .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
+ } catch (ConsumeQueueException e) {
+ throw new RemotingCommandException("Failed to query initial offset", e);
+ }
}
return -1L;
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
index d55f1b5b7fb..2f4cb7b15f8 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java
@@ -174,7 +174,14 @@ private Set doLoadBalance(final String topic, final String consume
break;
}
case CLUSTERING: {
- Set mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);
+ Set mqSet;
+ if (MixAll.isLmq(topic)) {
+ mqSet = new HashSet<>();
+ mqSet.add(new MessageQueue(
+ topic, brokerController.getBrokerConfig().getBrokerName(), (int)MixAll.LMQ_QUEUE_ID));
+ } else {
+ mqSet = topicRouteInfoManager.getTopicSubscribeInfo(topic);
+ }
if (null == mqSet) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java
new file mode 100644
index 00000000000..7a652f43151
--- /dev/null
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/RecallMessageProcessor.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageRequestHeader;
+import org.apache.rocketmq.remoting.protocol.header.RecallMessageResponseHeader;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+
+import java.nio.charset.StandardCharsets;
+
+public class RecallMessageProcessor implements NettyRequestProcessor {
+ private static final String RECALL_MESSAGE_TAG = "_RECALL_TAG_";
+ private final BrokerController brokerController;
+
+ public RecallMessageProcessor(final BrokerController brokerController) {
+ this.brokerController = brokerController;
+ }
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws
+ RemotingCommandException {
+ final RemotingCommand response = RemotingCommand.createResponseCommand(RecallMessageResponseHeader.class);
+ response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
+ final RecallMessageRequestHeader requestHeader =
+ request.decodeCommandCustomHeader(RecallMessageRequestHeader.class);
+
+ if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {
+ response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);
+ response.setRemark("recall failed, broker service not available");
+ return response;
+ }
+
+ final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
+ if (this.brokerController.getMessageStore().now() < startTimestamp) {
+ response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
+ response.setRemark("recall failed, broker service not available");
+ return response;
+ }
+
+ if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())
+ && !this.brokerController.getBrokerConfig().isAllowRecallWhenBrokerNotWriteable()) {
+ response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
+ response.setRemark("recall failed, broker service not available");
+ return response;
+ }
+
+ TopicConfig topicConfig =
+ this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+ if (null == topicConfig) {
+ response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+ response.setRemark("recall failed, the topic[" + requestHeader.getTopic() + "] not exist");
+ return response;
+ }
+
+ RecallMessageHandle.HandleV1 handle;
+ try {
+ handle = (RecallMessageHandle.HandleV1) RecallMessageHandle.decodeHandle(requestHeader.getRecallHandle());
+ } catch (DecoderException e) {
+ response.setCode(ResponseCode.ILLEGAL_OPERATION);
+ response.setRemark(e.getMessage());
+ return response;
+ }
+
+ if (!requestHeader.getTopic().equals(handle.getTopic())) {
+ response.setCode(ResponseCode.ILLEGAL_OPERATION);
+ response.setRemark("recall failed, topic not match");
+ return response;
+ }
+ if (!brokerController.getBrokerConfig().getBrokerName().equals(handle.getBrokerName())) {
+ response.setCode(ResponseCode.ILLEGAL_OPERATION);
+ response.setRemark("recall failed, broker service not available");
+ return response;
+ }
+
+ long timestamp = NumberUtils.toLong(handle.getTimestampStr(), -1);
+ long timeLeft = timestamp - System.currentTimeMillis();
+ if (timeLeft <= 0
+ || timeLeft >= brokerController.getMessageStoreConfig().getTimerMaxDelaySec() * 1000L) {
+ response.setCode(ResponseCode.ILLEGAL_OPERATION);
+ response.setRemark("recall failed, timestamp invalid");
+ return response;
+ }
+
+ MessageExtBrokerInner msgInner = buildMessage(ctx, requestHeader, handle);
+ long beginTimeMillis = this.brokerController.getMessageStore().now();
+ PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+ handlePutMessageResult(putMessageResult, request, response, msgInner, ctx, beginTimeMillis);
+ return response;
+ }
+
+ public MessageExtBrokerInner buildMessage(ChannelHandlerContext ctx, RecallMessageRequestHeader requestHeader,
+ RecallMessageHandle.HandleV1 handle) {
+ MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+ msgInner.setTopic(handle.getTopic());
+ msgInner.setBody("0".getBytes(StandardCharsets.UTF_8));
+ msgInner.setTags(RECALL_MESSAGE_TAG);
+ msgInner.setTagsCode(RECALL_MESSAGE_TAG.hashCode());
+ msgInner.setQueueId(0);
+ MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TIMER_DEL_UNIQKEY,
+ TimerMessageStore.buildDeleteKey(handle.getTopic(), handle.getMessageId()));
+ MessageAccessor.putProperty(msgInner,
+ MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, handle.getMessageId());
+ MessageAccessor.putProperty(msgInner,
+ MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(handle.getTimestampStr()));
+ MessageAccessor.putProperty(msgInner,
+ MessageConst.PROPERTY_BORN_TIMESTAMP, String.valueOf(System.currentTimeMillis()));
+ MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRACE_CONTEXT, "");
+ MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_PRODUCER_GROUP, requestHeader.getProducerGroup());
+ msgInner.setBornTimestamp(System.currentTimeMillis());
+ msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+ msgInner.setBornHost(ctx.channel().remoteAddress());
+ msgInner.setStoreHost(this.brokerController.getStoreHost());
+ return msgInner;
+ }
+
+ public void handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand request,
+ RemotingCommand response, MessageExt message, ChannelHandlerContext ctx, long beginTimeMillis) {
+ if (null == putMessageResult) {
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("recall failed, execute error");
+ return;
+ }
+ RecallMessageResponseHeader responseHeader = (RecallMessageResponseHeader) response.readCustomHeader();
+ switch (putMessageResult.getPutMessageStatus()) {
+ case PUT_OK:
+ this.brokerController.getBrokerStatsManager().incTopicPutNums(
+ message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); // system timer topic
+ this.brokerController.getBrokerStatsManager().incTopicPutSize(
+ message.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+ this.brokerController.getBrokerStatsManager().incBrokerPutNums(
+ message.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum());
+ this.brokerController.getBrokerStatsManager().incTopicPutLatency(
+ message.getTopic(), 0, (int) (this.brokerController.getMessageStore().now() - beginTimeMillis));
+ case FLUSH_DISK_TIMEOUT:
+ case FLUSH_SLAVE_TIMEOUT:
+ case SLAVE_NOT_AVAILABLE:
+ response.setCode(ResponseCode.SUCCESS);
+ responseHeader.setMsgId(MessageClientIDSetter.getUniqID(message));
+ break;
+ default:
+ response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setRemark("recall failed, execute error");
+ break;
+ }
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
index d3bb048f75d..a70b48debe1 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java
@@ -115,10 +115,10 @@ private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext c
response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
log.debug("receive SendReplyMessage request command, {}", request);
- final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
- if (this.brokerController.getMessageStore().now() < startTimstamp) {
+ final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
+ if (this.brokerController.getMessageStore().now() < startTimestamp) {
response.setCode(ResponseCode.SYSTEM_ERROR);
- response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp)));
+ response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimestamp)));
return response;
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
index 912d502eab2..669cd5e6771 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
@@ -40,6 +40,7 @@
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBatch;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.producer.RecallMessageHandle;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.common.utils.CleanupPolicyUtils;
@@ -430,7 +431,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult
"the broker's disk is full [" + diskUtil() + "], messages are put to the slave, message store has been shut down, etc.");
break;
case OS_PAGE_CACHE_BUSY:
- response.setCode(ResponseCode.SYSTEM_ERROR);
+ response.setCode(ResponseCode.SYSTEM_BUSY);
response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while");
break;
case LMQ_CONSUME_QUEUE_NUM_EXCEEDED:
@@ -483,6 +484,7 @@ private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult
responseHeader.setQueueId(queueIdInt);
responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());
responseHeader.setTransactionId(MessageClientIDSetter.getUniqID(msg));
+ attachRecallHandle(request, msg, responseHeader);
RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext);
if (rewriteResult != null) {
@@ -647,6 +649,21 @@ private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx,
}
}
+ public void attachRecallHandle(RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader) {
+ if (RequestCode.SEND_BATCH_MESSAGE == request.getCode()
+ || RequestCode.CONSUMER_SEND_MSG_BACK == request.getCode()) {
+ return;
+ }
+ String timestampStr = msg.getProperty(MessageConst.PROPERTY_TIMER_OUT_MS);
+ String realTopic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC);
+ if (timestampStr != null && realTopic != null && !realTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+ timestampStr = String.valueOf(Long.parseLong(timestampStr) + 1); // consider of floor
+ String recallHandle = RecallMessageHandle.HandleV1.buildHandle(realTopic,
+ brokerController.getBrokerConfig().getBrokerName(), timestampStr, MessageClientIDSetter.getUniqID(msg));
+ responseHeader.setRecallHandle(recallHandle);
+ }
+ }
+
private String diskUtil() {
double physicRatio = 100;
String storePath;
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
index e13b36df910..70184e8a620 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
@@ -53,6 +53,7 @@
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.queue.ReferredIterator;
@@ -103,7 +104,7 @@ public static int delayLevel2QueueId(final int delayLevel) {
return delayLevel - 1;
}
- public void buildRunningStats(HashMap stats) {
+ public void buildRunningStats(HashMap stats) throws ConsumeQueueException {
for (Map.Entry next : this.offsetTable.entrySet()) {
int queueId = delayLevel2QueueId(next.getKey());
long delayOffset = next.getValue();
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
index 7f802adb938..aa77b773ee9 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
@@ -17,8 +17,6 @@
package org.apache.rocketmq.broker.slave;
import java.io.IOException;
-import java.util.Iterator;
-import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
@@ -85,12 +83,7 @@ private void syncTopicConfig() {
ConcurrentMap newTopicConfigTable = topicWrapper.getTopicConfigTable();
//delete
ConcurrentMap topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable();
- for (Iterator> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) {
- Map.Entry item = it.next();
- if (!newTopicConfigTable.containsKey(item.getKey())) {
- it.remove();
- }
- }
+ topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey()));
//update
topicConfigTable.putAll(newTopicConfigTable);
@@ -104,12 +97,7 @@ private void syncTopicConfig() {
ConcurrentMap newTopicConfigTable = topicWrapper.getTopicConfigTable();
//delete
ConcurrentMap topicConfigTable = this.brokerController.getTopicConfigManager().getTopicConfigTable();
- for (Iterator> it = topicConfigTable.entrySet().iterator(); it.hasNext(); ) {
- Map.Entry item = it.next();
- if (!newTopicConfigTable.containsKey(item.getKey())) {
- it.remove();
- }
- }
+ topicConfigTable.entrySet().removeIf(item -> !newTopicConfigTable.containsKey(item.getKey()));
//update
topicConfigTable.putAll(newTopicConfigTable);
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
index 018083811e8..69e59fd8e7f 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/LmqSubscriptionGroupManager.java
@@ -43,4 +43,13 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config)
}
super.updateSubscriptionGroupConfig(config);
}
+
+ @Override
+ public boolean containsSubscriptionGroup(String group) {
+ if (MixAll.isLmq(group)) {
+ return true;
+ } else {
+ return super.containsSubscriptionGroup(group);
+ }
+ }
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
deleted file mode 100644
index e9a81a8d686..00000000000
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.rocketmq.broker.subscription;
-
-import java.io.File;
-
-import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
-import org.apache.rocketmq.common.utils.DataConverter;
-import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-
-public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
-
- public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
- super(brokerController, false);
- this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
- }
-
- @Override
- public boolean load() {
- if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) {
- return false;
- }
- this.init();
- return true;
- }
-
- @Override
- public boolean stop() {
- return this.rocksDBConfigManager.stop();
- }
-
- @Override
- protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
- String groupName = subscriptionGroupConfig.getGroupName();
- SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
-
- try {
- byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
- byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
- this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
- } catch (Exception e) {
- log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
- }
- return oldConfig;
- }
-
- @Override
- protected SubscriptionGroupConfig putSubscriptionGroupConfigIfAbsent(SubscriptionGroupConfig subscriptionGroupConfig) {
- String groupName = subscriptionGroupConfig.getGroupName();
- SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig);
- if (oldConfig == null) {
- try {
- byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8);
- byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible);
- this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes);
- } catch (Exception e) {
- log.error("kv put sub Failed, {}", subscriptionGroupConfig.toString());
- }
- }
- return oldConfig;
- }
-
- @Override
- protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) {
- SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName);
- try {
- this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8));
- } catch (Exception e) {
- log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString());
- }
- return subscriptionGroupConfig;
- }
-
- @Override
- protected void decode0(byte[] key, byte[] body) {
- String groupName = new String(key, DataConverter.CHARSET_UTF8);
- SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class);
-
- this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig);
- log.info("load exist local sub, {}", subscriptionGroupConfig.toString());
- }
-
- @Override
- public synchronized void persist() {
- if (brokerController.getMessageStoreConfig().isRealTimePersistRocksDBConfig()) {
- this.rocksDBConfigManager.flushWAL();
- }
- }
-
- @Override
- public String configFilePath() {
- return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "subscriptionGroups" + File.separator;
- }
-}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
index e63b9305868..f62a3e4a091 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java
@@ -19,6 +19,7 @@
import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
@@ -48,7 +49,7 @@ public class SubscriptionGroupManager extends ConfigManager {
private ConcurrentMap> forbiddenTable =
new ConcurrentHashMap<>(4);
- private final DataVersion dataVersion = new DataVersion();
+ protected final DataVersion dataVersion = new DataVersion();
protected transient BrokerController brokerController;
public SubscriptionGroupManager() {
@@ -121,7 +122,7 @@ protected void init() {
}
}
- protected SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
+ public SubscriptionGroupConfig putSubscriptionGroupConfig(SubscriptionGroupConfig subscriptionGroupConfig) {
return this.subscriptionGroupTable.put(subscriptionGroupConfig.getGroupName(), subscriptionGroupConfig);
}
@@ -138,6 +139,11 @@ protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName
}
public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {
+ updateSubscriptionGroupConfigWithoutPersist(config);
+ this.persist();
+ }
+
+ protected void updateSubscriptionGroupConfigWithoutPersist(SubscriptionGroupConfig config) {
Map newAttributes = request(config);
Map currentAttributes = current(config.getGroupName());
@@ -156,9 +162,14 @@ public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config)
log.info("create new subscription group, {}", config);
}
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
+ }
+ public void updateSubscriptionGroupConfigList(List configList) {
+ if (null == configList || configList.isEmpty()) {
+ return;
+ }
+ configList.forEach(this::updateSubscriptionGroupConfigWithoutPersist);
this.persist();
}
@@ -214,7 +225,7 @@ public int getForbidden(String group, String topic) {
return topicForbidden;
}
- private void updateForbiddenValue(String group, String topic, Integer forbidden) {
+ protected void updateForbiddenValue(String group, String topic, Integer forbidden) {
if (forbidden == null || forbidden <= 0) {
this.forbiddenTable.remove(group);
log.info("clear group forbidden, {}@{} ", group, topic);
@@ -233,8 +244,7 @@ private void updateForbiddenValue(String group, String topic, Integer forbidden)
log.info("set group forbidden, {}@{} old: {} new: {}", group, topic, 0, forbidden);
}
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
}
@@ -243,8 +253,7 @@ public void disableConsume(final String groupName) {
SubscriptionGroupConfig old = getSubscriptionGroupConfig(groupName);
if (old != null) {
old.setConsumeEnable(false);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
}
}
@@ -261,8 +270,7 @@ public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) {
if (null == preConfig) {
log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString());
}
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
}
}
@@ -326,13 +334,32 @@ public DataVersion getDataVersion() {
return dataVersion;
}
+ public boolean loadDataVersion() {
+ String fileName = null;
+ try {
+ fileName = this.configFilePath();
+ String jsonString = MixAll.file2String(fileName);
+ if (jsonString != null) {
+ SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class);
+ if (obj != null) {
+ this.dataVersion.assignNewOne(obj.dataVersion);
+ this.printLoadDataWhenFirstBoot(obj);
+ log.info("load subGroup dataVersion success,{},{}", fileName, obj.dataVersion);
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ log.error("load subGroup dataVersion failed" + fileName, e);
+ return false;
+ }
+ }
+
public void deleteSubscriptionGroupConfig(final String groupName) {
SubscriptionGroupConfig old = removeSubscriptionGroupConfig(groupName);
this.forbiddenTable.remove(groupName);
if (old != null) {
log.info("delete subscription group OK, subscription group:{}", old);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
} else {
log.warn("delete subscription group failed, subscription groupName: {} not exist", groupName);
@@ -369,4 +396,14 @@ private Map current(String groupName) {
}
}
}
+
+ public void setDataVersion(DataVersion dataVersion) {
+ this.dataVersion.assignNewOne(dataVersion);
+ }
+
+ public void updateDataVersion() {
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ dataVersion.nextVersion(stateMachineVersion);
+ }
+
}
diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
index 511d29e12ad..4530c10002d 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
@@ -18,6 +18,7 @@
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -51,6 +52,10 @@
import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper;
import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo;
+import org.apache.rocketmq.store.timer.TimerMessageStore;
+import org.apache.rocketmq.tieredstore.TieredMessageStore;
+import org.apache.rocketmq.tieredstore.metadata.MetadataStore;
+import org.apache.rocketmq.tieredstore.metadata.entity.TopicMetadata;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -61,7 +66,7 @@ public class TopicConfigManager extends ConfigManager {
private transient final Lock topicConfigTableLock = new ReentrantLock();
protected ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(1024);
- private DataVersion dataVersion = new DataVersion();
+ protected DataVersion dataVersion = new DataVersion();
protected transient BrokerController brokerController;
public TopicConfigManager() {
@@ -207,9 +212,20 @@ protected void init() {
topicConfig.setWriteQueueNums(1);
putTopicConfig(topicConfig);
}
+
+ {
+ if (this.brokerController.getMessageStoreConfig().isTimerWheelEnable()) {
+ String topic = TimerMessageStore.TIMER_TOPIC;
+ TopicConfig topicConfig = new TopicConfig(topic);
+ TopicValidator.addSystemTopic(topic);
+ topicConfig.setReadQueueNums(1);
+ topicConfig.setWriteQueueNums(1);
+ this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+ }
+ }
}
- protected TopicConfig putTopicConfig(TopicConfig topicConfig) {
+ public TopicConfig putTopicConfig(TopicConfig topicConfig) {
return this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
}
@@ -277,8 +293,7 @@ public TopicConfig createTopicInSendMessageMethod(final String topic, final Stri
putTopicConfig(topicConfig);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
createNew = true;
@@ -321,8 +336,7 @@ public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register
}
log.info("Create new topic [{}] config:[{}]", topicConfig.getTopicName(), topicConfig);
putTopicConfig(topicConfig);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
createNew = true;
this.persist();
} finally {
@@ -381,8 +395,7 @@ public TopicConfig createTopicInSendMessageBackMethod(
log.info("create new topic {}", topicConfig);
putTopicConfig(topicConfig);
createNew = true;
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
} finally {
this.topicConfigTableLock.unlock();
@@ -422,8 +435,7 @@ public TopicConfig createTopicOfTranCheckMaxTime(final int clientDefaultTopicQue
log.info("create new topic {}", topicConfig);
putTopicConfig(topicConfig);
createNew = true;
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
} finally {
this.topicConfigTableLock.unlock();
@@ -456,8 +468,7 @@ public void updateTopicUnitFlag(final String topic, final boolean unit) {
putTopicConfig(topicConfig);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
registerBrokerData(topicConfig);
@@ -479,21 +490,19 @@ public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub)
putTopicConfig(topicConfig);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
registerBrokerData(topicConfig);
}
}
- public void updateTopicConfig(final TopicConfig topicConfig) {
+ protected void updateSingleTopicConfigWithoutPersist(final TopicConfig topicConfig) {
checkNotNull(topicConfig, "topicConfig shouldn't be null");
Map newAttributes = request(topicConfig);
Map currentAttributes = current(topicConfig.getTopicName());
-
Map finalAttributes = AttributeUtil.alterCurrentAttributes(
this.topicConfigTable.get(topicConfig.getTopicName()) == null,
TopicAttributes.ALL,
@@ -501,6 +510,7 @@ public void updateTopicConfig(final TopicConfig topicConfig) {
ImmutableMap.copyOf(newAttributes));
topicConfig.setAttributes(finalAttributes);
+ updateTieredStoreTopicMetadata(topicConfig, newAttributes);
TopicConfig old = putTopicConfig(topicConfig);
if (old != null) {
@@ -509,12 +519,46 @@ public void updateTopicConfig(final TopicConfig topicConfig) {
log.info("create new topic [{}]", topicConfig);
}
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
+ }
+ public void updateTopicConfig(final TopicConfig topicConfig) {
+ updateSingleTopicConfigWithoutPersist(topicConfig);
this.persist(topicConfig.getTopicName(), topicConfig);
}
+ public void updateTopicConfigList(final List topicConfigList) {
+ topicConfigList.forEach(this::updateSingleTopicConfigWithoutPersist);
+ this.persist();
+ }
+
+ private synchronized void updateTieredStoreTopicMetadata(final TopicConfig topicConfig, Map newAttributes) {
+ if (!(brokerController.getMessageStore() instanceof TieredMessageStore)) {
+ if (newAttributes.get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName()) != null) {
+ throw new IllegalArgumentException("Update topic reserveTime not supported");
+ }
+ return;
+ }
+
+ String topic = topicConfig.getTopicName();
+ long reserveTime = TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getDefaultValue();
+ String attr = topicConfig.getAttributes().get(TopicAttributes.TOPIC_RESERVE_TIME_ATTRIBUTE.getName());
+ if (attr != null) {
+ reserveTime = Long.parseLong(attr);
+ }
+
+ log.info("Update tiered storage metadata, topic {}, reserveTime {}", topic, reserveTime);
+ TieredMessageStore tieredMessageStore = (TieredMessageStore) brokerController.getMessageStore();
+ MetadataStore metadataStore = tieredMessageStore.getMetadataStore();
+ TopicMetadata topicMetadata = metadataStore.getTopic(topic);
+ if (topicMetadata == null) {
+ metadataStore.addTopic(topic, reserveTime);
+ } else if (topicMetadata.getReserveTime() != reserveTime) {
+ topicMetadata.setReserveTime(reserveTime);
+ metadataStore.updateTopic(topicMetadata);
+ }
+ }
+
public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {
if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) {
@@ -529,25 +573,8 @@ public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {
}
}
- // We don't have a mandatory rule to maintain the validity of order conf in NameServer,
- // so we may overwrite the order field mistakenly.
- // To avoid the above case, we comment the below codes, please use mqadmin API to update
- // the order filed.
- /*for (Map.Entry entry : this.topicConfigTable.entrySet()) {
- String topic = entry.getKey();
- if (!orderTopics.contains(topic)) {
- TopicConfig topicConfig = entry.getValue();
- if (topicConfig.isOrder()) {
- topicConfig.setOrder(false);
- isChange = true;
- log.info("update order topic config, topic={}, order={}", topic, false);
- }
- }
- }*/
-
if (isChange) {
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
}
}
@@ -571,8 +598,7 @@ public void deleteTopicConfig(final String topic) {
TopicConfig old = removeTopicConfig(topic);
if (old != null) {
log.info("delete topic config OK, topic: {}", old);
- long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
- dataVersion.nextVersion(stateMachineVersion);
+ updateDataVersion();
this.persist();
} else {
log.warn("delete topic config failed, topic: {} not exists", topic);
@@ -611,6 +637,26 @@ public String encode() {
return encode(false);
}
+ public boolean loadDataVersion() {
+ String fileName = null;
+ try {
+ fileName = this.configFilePath();
+ String jsonString = MixAll.file2String(fileName);
+ if (jsonString != null) {
+ TopicConfigSerializeWrapper topicConfigSerializeWrapper =
+ TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class);
+ if (topicConfigSerializeWrapper != null) {
+ this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion());
+ log.info("load topic metadata dataVersion success {}, {}", fileName, topicConfigSerializeWrapper.getDataVersion());
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ log.error("load topic metadata dataVersion failed" + fileName, e);
+ return false;
+ }
+ }
+
@Override
public String configFilePath() {
return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
@@ -687,5 +733,11 @@ public boolean containsTopic(String topic) {
return topicConfigTable.containsKey(topic);
}
+ public void updateDataVersion() {
+ long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+ dataVersion.nextVersion(stateMachineVersion);
+ }
+
+
}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
index 8f89c14ae95..1d12acd4a98 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
@@ -20,29 +20,43 @@
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import io.netty.channel.DefaultChannelPromise;
+import io.netty.util.concurrent.DefaultEventExecutor;
+import org.apache.commons.lang3.tuple.Triple;
import org.apache.rocketmq.auth.config.AuthConfig;
import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.BrokerIdentity;
import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.netty.ResponseFuture;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.remoting.protocol.RequestCode;
import org.apache.rocketmq.remoting.protocol.ResponseCode;
import org.apache.rocketmq.remoting.protocol.body.ClusterInfo;
import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.remoting.protocol.header.PullMessageResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.namesrv.QueryDataVersionResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader;
import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult;
@@ -56,9 +70,12 @@
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
-import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -69,9 +86,11 @@
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.AdditionalMatchers.or;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;
-@RunWith(MockitoJUnitRunner.class)
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(NettyRemotingClient.class)
public class BrokerOuterAPITest {
@Mock
private ChannelHandlerContext handlerContext;
@@ -117,6 +136,9 @@ public void test_needRegister_normal() throws Exception {
@Test
public void test_needRegister_timeout() throws Exception {
+ if (MixAll.isMac()) {
+ return;
+ }
init();
brokerOuterAPI.start();
@@ -251,4 +273,154 @@ public void testLookupAddressByDomain() throws Exception {
});
Assert.assertTrue(result.get());
}
+
+ @Test
+ public void testPullMessageFromSpecificBrokerAsync_createChannel_null() throws Exception {
+ NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+ PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(null);
+ BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+ Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+ field.setAccessible(true);
+ field.set(api, mockClient);
+
+ Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+ Assert.assertNull(rst.getLeft());
+ Assert.assertTrue(rst.getMiddle().contains("connect"));
+ Assert.assertTrue(rst.getRight()); // need retry
+ }
+
+ @Test
+ public void testPullMessageFromSpecificBrokerAsync_createChannel_future_notSuccess() throws Exception {
+ NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+ DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+ PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+ BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+ Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+ field.setAccessible(true);
+ field.set(api, mockClient);
+
+ promise.tryFailure(new Throwable());
+ Triple rst
+ = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+ Assert.assertNull(rst.getLeft());
+ Assert.assertTrue(rst.getMiddle().contains("connect"));
+ Assert.assertTrue(rst.getRight()); // need retry
+ }
+
+ // skip other future status test
+
+ @Test
+ public void testPullMessageFromSpecificBrokerAsync_timeout() throws Exception {
+ Channel channel = Mockito.mock(Channel.class);
+ when(channel.isActive()).thenReturn(true);
+ NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+ DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+ PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+ when(promise.channel()).thenReturn(channel);
+ BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+ Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+ field.setAccessible(true);
+ field.set(api, mockClient);
+
+ CompletableFuture future = new CompletableFuture<>();
+ doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+ promise.trySuccess(null);
+ future.completeExceptionally(new RemotingTimeoutException("wait response on the channel timeout"));
+ Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+ Assert.assertNull(rst.getLeft());
+ Assert.assertTrue(rst.getMiddle().contains("timeout"));
+ Assert.assertTrue(rst.getRight()); // need retry
+ }
+
+ @Test
+ public void testPullMessageFromSpecificBrokerAsync_brokerReturn_pullStatusCode() throws Exception {
+ Channel channel = Mockito.mock(Channel.class);
+ when(channel.isActive()).thenReturn(true);
+ NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+ DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+ PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+ when(promise.channel()).thenReturn(channel);
+ BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+ Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+ field.setAccessible(true);
+ field.set(api, mockClient);
+
+ int[] respCodes = new int[] {ResponseCode.SUCCESS, ResponseCode.PULL_NOT_FOUND, ResponseCode.PULL_RETRY_IMMEDIATELY, ResponseCode.PULL_OFFSET_MOVED};
+ PullStatus[] respStatus = new PullStatus[] {PullStatus.FOUND, PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG, PullStatus.OFFSET_ILLEGAL};
+ for (int i = 0; i < respCodes.length; i++) {
+ CompletableFuture future = new CompletableFuture<>();
+ doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+ RemotingCommand response = mockPullMessageResponse(respCodes[i]);
+ ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,
+ resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+ responseFuture.setResponseCommand(response);
+ promise.trySuccess(null);
+ future.complete(responseFuture);
+
+ Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+ Assert.assertEquals(respStatus[i], rst.getLeft().getPullStatus());
+ if (ResponseCode.SUCCESS == respCodes[i]) {
+ Assert.assertEquals(1, rst.getLeft().getMsgFoundList().size());
+ } else {
+ Assert.assertNull(rst.getLeft().getMsgFoundList());
+ }
+ Assert.assertEquals(respStatus[i].name(), rst.getMiddle());
+ Assert.assertFalse(rst.getRight()); // no retry
+ }
+ }
+
+ @Test
+ public void testPullMessageFromSpecificBrokerAsync_brokerReturn_allOtherResponseCode() throws Exception {
+ Channel channel = Mockito.mock(Channel.class);
+ when(channel.isActive()).thenReturn(true);
+ NettyRemotingClient mockClient = PowerMockito.spy(new NettyRemotingClient(new NettyClientConfig()));
+ DefaultChannelPromise promise = PowerMockito.spy(new DefaultChannelPromise(PowerMockito.mock(Channel.class), new DefaultEventExecutor()));
+ PowerMockito.when(mockClient, "getAndCreateChannelAsync", any()).thenReturn(promise);
+ when(promise.channel()).thenReturn(channel);
+ BrokerOuterAPI api = new BrokerOuterAPI(new NettyClientConfig(), new AuthConfig());
+ Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
+ field.setAccessible(true);
+ field.set(api, mockClient);
+
+ CompletableFuture future = new CompletableFuture<>();
+ doReturn(future).when(mockClient).invokeImpl(any(Channel.class), any(RemotingCommand.class), anyLong());
+ // test one code here, skip others
+ RemotingCommand response = mockPullMessageResponse(ResponseCode.SUBSCRIPTION_NOT_EXIST);
+ ResponseFuture responseFuture = new ResponseFuture(channel, 0, null, 1000,
+ resp -> { }, new SemaphoreReleaseOnlyOnce(new Semaphore(1)));
+ responseFuture.setResponseCommand(response);
+ promise.trySuccess(null);
+ future.complete(responseFuture);
+
+ Triple rst = api.pullMessageFromSpecificBrokerAsync("", "", "", "", 1, 1, 1, 3000L).join();
+ Assert.assertNull(rst.getLeft());
+ Assert.assertTrue(rst.getMiddle().contains(ResponseCode.SUBSCRIPTION_NOT_EXIST + ""));
+ Assert.assertTrue(rst.getRight()); // need retry
+ }
+
+ private RemotingCommand mockPullMessageResponse(int responseCode) throws Exception {
+ RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
+ response.setCode(responseCode);
+ if (responseCode == ResponseCode.SUCCESS) {
+ MessageExt msg = new MessageExt();
+ msg.setBody("HW".getBytes());
+ msg.setTopic("topic");
+ msg.setBornHost(new InetSocketAddress("127.0.0.1", 9000));
+ msg.setStoreHost(new InetSocketAddress("127.0.0.1", 9000));
+ byte[] encode = MessageDecoder.encode(msg, false);
+ response.setBody(encode);
+ }
+ PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
+ responseHeader.setNextBeginOffset(0L);
+ responseHeader.setMaxOffset(0L);
+ responseHeader.setMinOffset(0L);
+ responseHeader.setOffsetDelta(0L);
+ responseHeader.setTopicSysFlag(0);
+ responseHeader.setGroupSysFlag(0);
+ responseHeader.setSuggestWhichBrokerId(0L);
+ responseHeader.setForbiddenType(0);
+ response.makeCustomHeaderToNet();
+ return response;
+ }
+
}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
index 8c909824348..a23ad20037c 100644
--- a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerTest.java
@@ -18,10 +18,7 @@
package org.apache.rocketmq.broker.client;
import io.netty.channel.Channel;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.broker.client.net.Broker2Client;
import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@ -30,13 +27,25 @@
import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
-import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -49,19 +58,10 @@ public class ConsumerManagerTest {
private ConsumerManager consumerManager;
- private DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener;
-
@Mock
private BrokerController brokerController;
- @Mock
- private ConsumerFilterManager consumerFilterManager;
-
- private BrokerConfig brokerConfig = new BrokerConfig();
-
- private Broker2Client broker2Client;
-
- private BrokerStatsManager brokerStatsManager;
+ private final BrokerConfig brokerConfig = new BrokerConfig();
private static final String GROUP = "DEFAULT_GROUP";
@@ -74,40 +74,38 @@ public class ConsumerManagerTest {
@Before
public void before() {
clientChannelInfo = new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, VERSION);
- defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController);
- brokerStatsManager = new BrokerStatsManager(brokerConfig);
- consumerManager = new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig);
- broker2Client = new Broker2Client(brokerController);
+ DefaultConsumerIdsChangeListener defaultConsumerIdsChangeListener = new DefaultConsumerIdsChangeListener(brokerController);
+ BrokerStatsManager brokerStatsManager = new BrokerStatsManager(brokerConfig);
+ consumerManager = spy(new ConsumerManager(defaultConsumerIdsChangeListener, brokerStatsManager, brokerConfig));
+ ConsumerFilterManager consumerFilterManager = mock(ConsumerFilterManager.class);
when(brokerController.getConsumerFilterManager()).thenReturn(consumerFilterManager);
- when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
- when(brokerController.getBroker2Client()).thenReturn(broker2Client);
}
@Test
public void compensateBasicConsumerInfoTest() {
ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
- Assertions.assertThat(consumerGroupInfo).isNull();
+ assertThat(consumerGroupInfo).isNull();
consumerManager.compensateBasicConsumerInfo(GROUP, ConsumeType.CONSUME_ACTIVELY, MessageModel.BROADCASTING);
consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
- Assertions.assertThat(consumerGroupInfo).isNotNull();
- Assertions.assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);
- Assertions.assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING);
+ assertThat(consumerGroupInfo).isNotNull();
+ assertThat(consumerGroupInfo.getConsumeType()).isEqualTo(ConsumeType.CONSUME_ACTIVELY);
+ assertThat(consumerGroupInfo.getMessageModel()).isEqualTo(MessageModel.BROADCASTING);
}
@Test
public void compensateSubscribeDataTest() {
ConsumerGroupInfo consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
- Assertions.assertThat(consumerGroupInfo).isNull();
+ assertThat(consumerGroupInfo).isNull();
consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
consumerGroupInfo = consumerManager.getConsumerGroupInfo(GROUP, true);
- Assertions.assertThat(consumerGroupInfo).isNotNull();
- Assertions.assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1);
+ assertThat(consumerGroupInfo).isNotNull();
+ assertThat(consumerGroupInfo.getSubscriptionTable().size()).isEqualTo(1);
SubscriptionData subscriptionData = consumerGroupInfo.getSubscriptionTable().get(TOPIC);
- Assertions.assertThat(subscriptionData).isNotNull();
- Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
- Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
+ assertThat(subscriptionData).isNotNull();
+ assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
+ assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
}
@Test
@@ -118,7 +116,8 @@ public void registerConsumerTest() {
subList.add(subscriptionData);
consumerManager.registerConsumer(GROUP, clientChannelInfo, ConsumeType.CONSUME_PASSIVELY,
MessageModel.BROADCASTING, ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET, subList, true);
- Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull();
+ verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+ assertThat(consumerManager.getConsumerTable().get(GROUP)).isNotNull();
}
@Test
@@ -128,63 +127,65 @@ public void unregisterConsumerTest() {
// unregister
consumerManager.unregisterConsumer(GROUP, clientChannelInfo, true);
- Assertions.assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull();
+ verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+ assertThat(consumerManager.getConsumerTable().get(GROUP)).isNull();
}
@Test
public void findChannelTest() {
register();
final ClientChannelInfo consumerManagerChannel = consumerManager.findChannel(GROUP, CLIENT_ID);
- Assertions.assertThat(consumerManagerChannel).isNotNull();
+ assertThat(consumerManagerChannel).isNotNull();
}
@Test
public void findSubscriptionDataTest() {
register();
final SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC);
- Assertions.assertThat(subscriptionData).isNotNull();
+ assertThat(subscriptionData).isNotNull();
}
@Test
public void findSubscriptionDataCountTest() {
register();
final int count = consumerManager.findSubscriptionDataCount(GROUP);
- assert count > 0;
+ assertTrue(count > 0);
}
@Test
public void findSubscriptionTest() {
SubscriptionData subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);
- Assertions.assertThat(subscriptionData).isNull();
+ assertThat(subscriptionData).isNull();
consumerManager.compensateSubscribeData(GROUP, TOPIC, new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, true);
- Assertions.assertThat(subscriptionData).isNotNull();
- Assertions.assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
- Assertions.assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
+ assertThat(subscriptionData).isNotNull();
+ assertThat(subscriptionData.getTopic()).isEqualTo(TOPIC);
+ assertThat(subscriptionData.getSubString()).isEqualTo(SubscriptionData.SUB_ALL);
subscriptionData = consumerManager.findSubscriptionData(GROUP, TOPIC, false);
- Assertions.assertThat(subscriptionData).isNull();
+ assertThat(subscriptionData).isNull();
}
@Test
public void scanNotActiveChannelTest() {
clientChannelInfo.setLastUpdateTimestamp(System.currentTimeMillis() - brokerConfig.getChannelExpiredTimeout() * 2);
consumerManager.scanNotActiveChannel();
- Assertions.assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0);
+ assertThat(consumerManager.getConsumerTable().size()).isEqualTo(0);
}
@Test
public void queryTopicConsumeByWhoTest() {
register();
final HashSet consumeGroup = consumerManager.queryTopicConsumeByWho(TOPIC);
- assert consumeGroup.size() > 0;
+ assertFalse(consumeGroup.isEmpty());
}
@Test
public void doChannelCloseEventTest() {
consumerManager.doChannelCloseEvent("127.0.0.1", channel);
- assert consumerManager.getConsumerTable().size() == 0;
+ verify(consumerManager, never()).callConsumerIdsChangeListener(eq(ConsumerGroupEvent.CHANGE), any(), any());
+ assertEquals(0, consumerManager.getConsumerTable().size());
}
private void register() {
@@ -203,8 +204,8 @@ public void removeExpireConsumerGroupInfo() {
consumerManager.compensateSubscribeData(GROUP, TOPIC, subscriptionData);
consumerManager.compensateSubscribeData(GROUP, TOPIC + "_1", new SubscriptionData(TOPIC, SubscriptionData.SUB_ALL));
consumerManager.removeExpireConsumerGroupInfo();
- Assertions.assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull();
- Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull();
- Assertions.assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull();
+ assertThat(consumerManager.getConsumerGroupInfo(GROUP, true)).isNotNull();
+ assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC)).isNull();
+ assertThat(consumerManager.findSubscriptionData(GROUP, TOPIC + "_1")).isNotNull();
}
}
diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java
new file mode 100644
index 00000000000..ccb489aead2
--- /dev/null
+++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client.net;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.client.ConsumerManager;
+import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.RemotingServer;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.apache.rocketmq.remoting.protocol.ResponseCode;
+import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody;
+import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader;
+import org.apache.rocketmq.store.MessageStore;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class Broker2ClientTest {
+
+ @Mock
+ private BrokerController brokerController;
+
+ @Mock
+ private RemotingServer remotingServer;
+
+ @Mock
+ private ConsumerManager consumerManager;
+
+ @Mock
+ private TopicConfigManager topicConfigManager;
+
+ @Mock
+ private ConsumerOffsetManager consumerOffsetManager;
+
+ @Mock
+ private Channel channel;
+
+ @Mock
+ private ConsumerGroupInfo consumerGroupInfo;
+
+ private Broker2Client broker2Client;
+
+ private final String defaultTopic = "defaultTopic";
+
+ private final String defaultBroker = "defaultBroker";
+
+ private final String defaultGroup = "defaultGroup";
+
+ private final long timestamp = System.currentTimeMillis();
+
+ private final boolean isForce = true;
+
+ @Before
+ public void init() {
+ broker2Client = new Broker2Client(brokerController);
+ when(brokerController.getRemotingServer()).thenReturn(remotingServer);
+ when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+ when(brokerController.getConsumerManager()).thenReturn(consumerManager);
+ when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager);
+ when(brokerController.getBrokerConfig()).thenReturn(mock(BrokerConfig.class));
+ when(brokerController.getMessageStore()).thenReturn(mock(MessageStore.class));
+ when(consumerManager.getConsumerGroupInfo(any())).thenReturn(consumerGroupInfo);
+ }
+
+ @Test
+ public void testCheckProducerTransactionState() throws Exception {
+ CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+ broker2Client.checkProducerTransactionState("group", channel, requestHeader, createMessageExt());
+ verify(remotingServer).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));
+ }
+
+ @Test
+ public void testCheckProducerTransactionStateException() throws Exception {
+ CheckTransactionStateRequestHeader requestHeader = new CheckTransactionStateRequestHeader();
+ MessageExt messageExt = createMessageExt();
+ doThrow(new RuntimeException("Test Exception"))
+ .when(remotingServer)
+ .invokeOneway(any(Channel.class),
+ any(RemotingCommand.class),
+ anyLong());
+ broker2Client.checkProducerTransactionState("group", channel, requestHeader, messageExt);
+ verify(brokerController.getRemotingServer()).invokeOneway(eq(channel), any(RemotingCommand.class), eq(10L));
+ }
+
+ @Test
+ public void testResetOffsetNoTopicConfig() throws RemotingCommandException {
+ when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null);
+ RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+ assertEquals(ResponseCode.TOPIC_NOT_EXIST, response.getCode());
+ }
+
+ @Test
+ public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException {
+ TopicConfig topicConfig = mock(TopicConfig.class);
+ when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
+ when(topicConfig.getWriteQueueNums()).thenReturn(1);
+ when(consumerOffsetManager.queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);
+ RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+ assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());
+ }
+
+ @Test
+ public void testResetOffset() throws RemotingCommandException {
+ TopicConfig topicConfig = mock(TopicConfig.class);
+ when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
+ when(topicConfig.getWriteQueueNums()).thenReturn(1);
+ when(brokerController.getConsumerOffsetManager().queryOffset(defaultGroup, defaultTopic, 0)).thenReturn(0L);
+ BrokerConfig brokerConfig = mock(BrokerConfig.class);
+ when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
+ when(brokerConfig.getBrokerName()).thenReturn(defaultBroker);
+ ConsumerGroupInfo consumerGroupInfo = mock(ConsumerGroupInfo.class);
+ when(consumerManager.getConsumerGroupInfo(defaultGroup)).thenReturn(consumerGroupInfo);
+ RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
+ assertEquals(ResponseCode.CONSUMER_NOT_ONLINE, response.getCode());
+ }
+
+ @Test
+ public void testGetConsumeStatusNoConsumerOnline() {
+ when(consumerGroupInfo.getChannelInfoTable()).thenReturn(new ConcurrentHashMap<>());
+ RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, "");
+ assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+ }
+
+ @Test
+ public void testGetConsumeStatusClientDoesNotSupportFeature() {
+ ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, "defaultClientId", null, MQVersion.Version.V3_0_6.ordinal());
+ ConcurrentMap channelInfoTable = new ConcurrentHashMap<>();
+ channelInfoTable.put(channel, clientChannelInfo);
+ when(consumerGroupInfo.getChannelInfoTable()).thenReturn(channelInfoTable);
+ RemotingCommand response = broker2Client.getConsumeStatus(defaultTopic, defaultGroup, "");
+ assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
+ }
+
+ @Test
+ public void testGetConsumeStatus() throws Exception {
+ ConcurrentMap