ICU-22642 Avoid spending too much time inside CanonicalIterator

See #3017

icu4c/source/common/caniter.cpp

@@ -363,6 +363,9 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
     char16_t USeg[256];
     int32_t segLen = segment.extract(USeg, 256, status);
     getEquivalents2(&basic, USeg, segLen, status);
+    if (U_FAILURE(status)) {
+        return nullptr;
+    }
 
     // now get all the permutations
     // add only the ones that are canonically equivalent
@@ -466,6 +469,9 @@ Hashtable *CanonicalIterator::getEquivalents2(Hashtable *fillinResult, const cha
             Hashtable remainder(status);
             remainder.setValueDeleter(uprv_deleteUObject);
             if (extract(&remainder, cp2, segment, segLen, i, status) == nullptr) {
+                if (U_FAILURE(status)) {
+                    return nullptr;
+                }
                 continue;
             }
 
@@ -490,6 +496,13 @@ Hashtable *CanonicalIterator::getEquivalents2(Hashtable *fillinResult, const cha
 
                 ne = remainder.nextElement(el);
             }
+            // ICU-22642 Guards against strings that have so many permutations
+            // that they would otherwise hang the function.
+            constexpr int32_t kResultLimit = 4096;
+            if (fillinResult->count() > kResultLimit) {
+                status = U_UNSUPPORTED_ERROR;
+                return nullptr;
+            }
         }
     }
 

icu4j/main/core/src/main/java/com/ibm/icu/text/CanonicalIterator.java

@@ -172,7 +172,7 @@ public static void permute(String source, boolean skipZeros, Set<String> output)
         permute(source, skipZeros, output, 0);
     }
 
-    private static int PERMUTE_DEPTH_LIMIT = 8;
+    private static final int PERMUTE_DEPTH_LIMIT = 8;
     /**
      * Simple implementation of permutation.
      * <br><b>Warning: The strings are not guaranteed to be in any particular order.</b>
@@ -308,6 +308,7 @@ private String[] getEquivalents(String segment) {
     }
 
 
+    static private final int RESULT_LIMIT = 4096;
     private Set<String> getEquivalents2(String segment) {
 
         Set<String> result = new HashSet<String>();
@@ -342,6 +343,12 @@ private Set<String> getEquivalents2(String segment) {
                     result.add(prefix + item);
                 }
             }
+
+            // ICU-22642 Guards against strings that have so many permutations
+            // that they would otherwise hang the function.
+            if (result.size() > RESULT_LIMIT) {
+                throw new UnsupportedOperationException("Too many permutations");
+            }
         }
         return result;
         /*