diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
index 12af3ee9a1e..e0461769568 100644
--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
+++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java
@@ -91,7 +91,7 @@ public void handle(ConsumerGroupEvent event, String group, Object... args) {
                             this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
                         }
                         if (isNormalCompletion) {
-                            activeGroupNotifyMap.remove(group);
+                            activeGroupNotifyMap.computeIfPresent(group, (k, val) -> val == currentNotifyTaskControl ? null : val);
                         }
                     } else {
                         consumerChannelMap.put(group, channels);