Skip to content

Commit

Permalink
[ISSUE #9111] Support export broker RocksDB Config to json file (#9114)
Browse files Browse the repository at this point in the history
* [ISSUE #9111] Broker Support export RocksDB Config to json file and enhance admin tools
  • Loading branch information
qianye1001 authored Jan 14, 2025
1 parent f6249e5 commit b30afe8
Show file tree
Hide file tree
Showing 13 changed files with 463 additions and 42 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager {

protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);

protected RocksDBConfigManager rocksDBConfigManager;
protected transient RocksDBConfigManager rocksDBConfigManager;

public RocksDBConsumerOffsetManager(BrokerController brokerController) {
super(brokerController);
Expand Down Expand Up @@ -100,7 +100,7 @@ protected void removeConsumerOffset(String topicAtGroup) {
byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8);
this.rocksDBConfigManager.delete(keyBytes);
} catch (Exception e) {
LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup);
log.error("kv remove consumerOffset Failed, {}", topicAtGroup);
}
}

Expand All @@ -109,7 +109,7 @@ protected void decodeOffset(final byte[] key, final byte[] body) {
RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class);

this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable());
LOG.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable());
log.info("load exist local offset, {}, {}", topicAtGroup, wrapper.getOffsetTable());
}

public String rocksdbConfigFilePath() {
Expand All @@ -132,12 +132,17 @@ public synchronized void persist() {
this.rocksDBConfigManager.batchPutWithWal(writeBatch);
this.rocksDBConfigManager.flushWAL();
} catch (Exception e) {
LOG.error("consumer offset persist Failed", e);
log.error("consumer offset persist Failed", e);
} finally {
writeBatch.close();
}
}

public synchronized void exportToJson() {
log.info("RocksDBConsumerOffsetManager export consumer offset to json file");
super.persist();
}

private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap<Integer, Long> offsetMap) throws Exception {
byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8);
RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {

protected RocksDBConfigManager rocksDBConfigManager;
protected transient RocksDBConfigManager rocksDBConfigManager;

public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
super(brokerController, false);
Expand Down Expand Up @@ -184,6 +184,11 @@ public synchronized void persist() {
}
}

public synchronized void exportToJson() {
log.info("RocksDBSubscriptionGroupManager export subscription group to json file");
super.persist();
}

public String rocksdbConfigFilePath() {
return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "subscriptionGroups" + File.separator;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

public class RocksDBTopicConfigManager extends TopicConfigManager {

protected RocksDBConfigManager rocksDBConfigManager;
protected transient RocksDBConfigManager rocksDBConfigManager;

public RocksDBTopicConfigManager(BrokerController brokerController) {
super(brokerController, false);
Expand Down Expand Up @@ -139,6 +139,11 @@ public synchronized void persist() {
}
}

public synchronized void exportToJson() {
log.info("RocksDBTopicConfigManager export topic config to json file");
super.persist();
}

public String rocksdbConfigFilePath() {
return this.brokerController.getMessageStoreConfig().getStorePathRootDir() + File.separator + "config" + File.separator + "topics" + File.separator;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
Expand All @@ -60,6 +61,9 @@
import org.apache.rocketmq.broker.auth.converter.UserConverter;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.config.v1.RocksDBConsumerOffsetManager;
import org.apache.rocketmq.broker.config.v1.RocksDBSubscriptionGroupManager;
import org.apache.rocketmq.broker.config.v1.RocksDBTopicConfigManager;
import org.apache.rocketmq.broker.controller.ReplicasManager;
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
Expand Down Expand Up @@ -159,6 +163,7 @@
import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ExchangeHAInfoResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.GetAllTopicConfigResponseHeader;
Expand Down Expand Up @@ -239,7 +244,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
protected final BrokerController brokerController;
protected Set<String> configBlackList = new HashSet<>();
private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
private final ExecutorService asyncExecuteWorker = new ThreadPoolExecutor(0, 4, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());

public AdminBrokerProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
Expand Down Expand Up @@ -356,6 +361,8 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx,
return queryConsumeQueue(ctx, request);
case RequestCode.CHECK_ROCKSDB_CQ_WRITE_PROGRESS:
return this.checkRocksdbCqWriteProgress(ctx, request);
case RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON:
return this.exportRocksDBConfigToJson(ctx, request);
case RequestCode.UPDATE_AND_GET_GROUP_FORBIDDEN:
return this.updateAndGetGroupForbidden(ctx, request);
case RequestCode.GET_SUBSCRIPTIONGROUP_CONFIG:
Expand Down Expand Up @@ -495,6 +502,51 @@ private RemotingCommand checkRocksdbCqWriteProgress(ChannelHandlerContext ctx, R
return response;
}

private RemotingCommand exportRocksDBConfigToJson(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
ExportRocksDBConfigToJsonRequestHeader requestHeader = request.decodeCommandCustomHeader(ExportRocksDBConfigToJsonRequestHeader.class);
List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = requestHeader.fetchConfigType();
List<CompletableFuture<Void>> futureList = new ArrayList<>(configTypes.size());
for (ExportRocksDBConfigToJsonRequestHeader.ConfigType type : configTypes) {
switch (type) {
case TOPICS:
if (this.brokerController.getTopicConfigManager() instanceof RocksDBTopicConfigManager) {
RocksDBTopicConfigManager rocksDBTopicConfigManager = (RocksDBTopicConfigManager) this.brokerController.getTopicConfigManager();
futureList.add(CompletableFuture.runAsync(rocksDBTopicConfigManager::exportToJson, asyncExecuteWorker));
}
break;
case SUBSCRIPTION_GROUPS:
if (this.brokerController.getSubscriptionGroupManager() instanceof RocksDBSubscriptionGroupManager) {
RocksDBSubscriptionGroupManager rocksDBSubscriptionGroupManager = (RocksDBSubscriptionGroupManager) this.brokerController.getSubscriptionGroupManager();
futureList.add(CompletableFuture.runAsync(rocksDBSubscriptionGroupManager::exportToJson, asyncExecuteWorker));
}
break;
case CONSUMER_OFFSETS:
if (this.brokerController.getConsumerOffsetManager() instanceof RocksDBConsumerOffsetManager) {
RocksDBConsumerOffsetManager rocksDBConsumerOffsetManager = (RocksDBConsumerOffsetManager) this.brokerController.getConsumerOffsetManager();
futureList.add(CompletableFuture.runAsync(rocksDBConsumerOffsetManager::exportToJson, asyncExecuteWorker));
}
break;
default:
break;
}
}

try {
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
} catch (CompletionException e) {
RemotingCommand response = RemotingCommand.createResponseCommand(null);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(String.valueOf(e));
return response;
}

RemotingCommand response = RemotingCommand.createResponseCommand(null);
response.setCode(ResponseCode.SUCCESS);
response.setRemark("export done.");
return response;
}

@Override
public boolean rejectRequest() {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
import org.apache.rocketmq.remoting.protocol.header.DeleteTopicRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.DeleteUserRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
import org.apache.rocketmq.remoting.protocol.header.GetAclRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.GetAllProducerInfoRequestHeader;
Expand Down Expand Up @@ -3036,6 +3037,21 @@ public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(final String broker
throw new MQClientException(response.getCode(), response.getRemark());
}

public void exportRocksDBConfigToJson(final String brokerAddr,
final List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType,
final long timeoutMillis) throws InterruptedException,
RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
ExportRocksDBConfigToJsonRequestHeader header = new ExportRocksDBConfigToJsonRequestHeader();
header.updateConfigType(configType);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, header);
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;

if (ResponseCode.SUCCESS != response.getCode()) {
throw new MQClientException(response.getCode(), response.getRemark());
}
}

public void checkClientInBroker(final String brokerAddr, final String consumerGroup,
final String clientId, final SubscriptionData subscriptionData,
final long timeoutMillis)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ public class RequestCode {
public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352;
public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353;
public static final int CHECK_ROCKSDB_CQ_WRITE_PROGRESS = 354;
public static final int EXPORT_ROCKSDB_CONFIG_TO_JSON = 355;

public static final int LITE_PULL_MESSAGE = 361;
public static final int RECALL_MESSAGE = 370;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.remoting.protocol.header;

import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.action.Action;
import org.apache.rocketmq.common.action.RocketMQAction;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.protocol.RequestCode;

@RocketMQAction(value = RequestCode.EXPORT_ROCKSDB_CONFIG_TO_JSON, action = Action.GET)
public class ExportRocksDBConfigToJsonRequestHeader implements CommandCustomHeader {
private static final String CONFIG_TYPE_SEPARATOR = ";";

public enum ConfigType {
TOPICS("topics"),
SUBSCRIPTION_GROUPS("subscriptionGroups"),
CONSUMER_OFFSETS("consumerOffsets");

private final String typeName;

ConfigType(String typeName) {
this.typeName = typeName;
}

public static ConfigType getConfigTypeByName(String typeName) {
for (ConfigType configType : ConfigType.values()) {
if (configType.getTypeName().equalsIgnoreCase(typeName.trim())) {
return configType;
}
}
throw new IllegalArgumentException("Unknown config type: " + typeName);
}

public static List<ConfigType> fromString(String ordinal) {
String[] configTypeNames = StringUtils.split(ordinal, CONFIG_TYPE_SEPARATOR);
List<ConfigType> configTypes = new ArrayList<>();
for (String configTypeName : configTypeNames) {
if (StringUtils.isNotEmpty(configTypeName)) {
configTypes.add(getConfigTypeByName(configTypeName));
}
}
return configTypes;
}

public static String toString(List<ConfigType> configTypes) {
StringBuilder sb = new StringBuilder();
for (ConfigType configType : configTypes) {
sb.append(configType.getTypeName()).append(CONFIG_TYPE_SEPARATOR);
}
return sb.toString();
}

public String getTypeName() {
return typeName;
}
}

@CFNotNull
private String configType;

@Override
public void checkFields() throws RemotingCommandException {

}

public List<ConfigType> fetchConfigType() {
return ConfigType.fromString(configType);
}

public void updateConfigType(List<ConfigType> configType) {
this.configType = ConfigType.toString(configType);
}

public String getConfigType() {
return configType;
}

public void setConfigType(String configType) {
this.configType = configType;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.remoting.protocol.header;

import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;

public class ExportRocksDBConfigToJsonRequestHeaderTest {
@Test
public void configTypeTest() {
List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configTypes = new ArrayList<>();
configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);
configTypes.add(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);

String string = ExportRocksDBConfigToJsonRequestHeader.ConfigType.toString(configTypes);

List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> newConfigTypes = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString(string);
assert newConfigTypes.size() == 2;
assert configTypes.equals(newConfigTypes);

List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> topics = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("topics");
assert topics.size() == 1;
assert topics.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);

List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> mix = ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("toPics; subScriptiongroups");
assert mix.size() == 2;
assert mix.get(0).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.TOPICS);
assert mix.get(1).equals(ExportRocksDBConfigToJsonRequestHeader.ConfigType.SUBSCRIPTION_GROUPS);

Assert.assertThrows(IllegalArgumentException.class, () -> {
ExportRocksDBConfigToJsonRequestHeader.ConfigType.fromString("topics; subscription");
});

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.remoting.protocol.body.TopicList;
import org.apache.rocketmq.remoting.protocol.body.UserInfo;
import org.apache.rocketmq.remoting.protocol.header.ExportRocksDBConfigToJsonRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader;
import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
Expand Down Expand Up @@ -778,6 +779,13 @@ public CheckRocksdbCqWriteResult checkRocksdbCqWriteProgress(String brokerAddr,
return this.defaultMQAdminExtImpl.checkRocksdbCqWriteProgress(brokerAddr, topic, checkStoreTime);
}

@Override
public void exportRocksDBConfigToJson(String brokerAddr,
List<ExportRocksDBConfigToJsonRequestHeader.ConfigType> configType)
throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException {
this.defaultMQAdminExtImpl.exportRocksDBConfigToJson(brokerAddr, configType);
}

@Override
public boolean resumeCheckHalfMessage(String topic,
String msgId)
Expand Down
Loading

0 comments on commit b30afe8

Please sign in to comment.