Merge pull request #40 from mcarvin8/export

Exports

src/index.ts

@@ -5,6 +5,8 @@ export { DisassembleXMLFileHandler } from "./service/disassembleXMLFileHandler";
 export { parseXML } from "./service/parseXML";
 export { buildXMLString } from "./service/buildXMLString";
 export { XmlElement } from "./helpers/types";
+export { getConcurrencyThreshold } from "./service/getConcurrencyThreshold";
+export { withConcurrencyLimit } from "./service/withConcurrencyLimit";
 
 // Function to update the log level
 export function setLogLevel(level: string) {

src/service/buildDisassembledFiles.ts

@@ -9,6 +9,7 @@ import { buildRootElementHeader } from "@src/service/buildRootElementHeader";
 import { buildLeafFile } from "@src/service/buildLeafFile";
 import { parseXML } from "@src/service/parseXML";
 import { getConcurrencyThreshold } from "./getConcurrencyThreshold";
+import { withConcurrencyLimit } from "./withConcurrencyLimit";
 
 export async function buildDisassembledFiles(
   filePath: string,
@@ -37,19 +38,38 @@ export async function buildDisassembledFiles(
   );
 
   const concurrencyLimit = getConcurrencyThreshold();
-  const activePromises: Promise<void>[] = [];
-  let currentIndex = 0;
 
-  const processChildKey = async (key: string) => {
-    if (Array.isArray(rootElement[key])) {
-      await Promise.all(
-        (rootElement[key] as XmlElement[]).map(async (element) => {
-          const [
-            updatedLeafContent,
-            updatedLeafCount,
-            updatedHasNestedElements,
-          ] = await processElement({
-            element,
+  // Create tasks for processing child keys
+  const tasks: (() => Promise<void>)[] = childKeys.map((key) => {
+    return async () => {
+      if (Array.isArray(rootElement[key])) {
+        await Promise.all(
+          (rootElement[key] as XmlElement[]).map(async (element) => {
+            const [
+              updatedLeafContent,
+              updatedLeafCount,
+              updatedHasNestedElements,
+            ] = await processElement({
+              element,
+              disassembledPath,
+              uniqueIdElements,
+              rootElementName,
+              rootElementHeader,
+              key,
+              indent,
+              leafContent: "",
+              leafCount: 0,
+              hasNestedElements: false,
+            });
+            leafContent += updatedLeafContent;
+            leafCount += updatedLeafCount;
+            hasNestedElements = hasNestedElements || updatedHasNestedElements;
+          }),
+        );
+      } else {
+        const [updatedLeafContent, updatedLeafCount, updatedHasNestedElements] =
+          await processElement({
+            element: rootElement[key] as XmlElement,
             disassembledPath,
             uniqueIdElements,
             rootElementName,
@@ -60,45 +80,15 @@ export async function buildDisassembledFiles(
             leafCount: 0,
             hasNestedElements: false,
           });
-          leafContent += updatedLeafContent;
-          leafCount += updatedLeafCount;
-          hasNestedElements = hasNestedElements || updatedHasNestedElements;
-        }),
-      );
-    } else {
-      const [updatedLeafContent, updatedLeafCount, updatedHasNestedElements] =
-        await processElement({
-          element: rootElement[key] as XmlElement,
-          disassembledPath,
-          uniqueIdElements,
-          rootElementName,
-          rootElementHeader,
-          key,
-          indent,
-          leafContent: "",
-          leafCount: 0,
-          hasNestedElements: false,
-        });
-      leafContent += updatedLeafContent;
-      leafCount += updatedLeafCount;
-      hasNestedElements = hasNestedElements || updatedHasNestedElements;
-    }
-  };
+        leafContent += updatedLeafContent;
+        leafCount += updatedLeafCount;
+        hasNestedElements = hasNestedElements || updatedHasNestedElements;
+      }
+    };
+  });
 
-  while (currentIndex < childKeys.length || activePromises.length > 0) {
-    if (
-      currentIndex < childKeys.length &&
-      activePromises.length < concurrencyLimit
-    ) {
-      const key = childKeys[currentIndex++];
-      const promise = processChildKey(key).finally(() => {
-        activePromises.splice(activePromises.indexOf(promise), 1);
-      });
-      activePromises.push(promise);
-    } else {
-      await Promise.race(activePromises);
-    }
-  }
+  // Execute tasks with concurrency limit
+  await withConcurrencyLimit(tasks, concurrencyLimit);
 
   if (!hasNestedElements) {
     logger.error(

src/service/disassembleXMLFileHandler.ts

@@ -10,6 +10,7 @@ import { logger } from "@src/index";
 import { INDENT } from "@src/helpers/constants";
 import { buildDisassembledFiles } from "@src/service/buildDisassembledFiles";
 import { getConcurrencyThreshold } from "./getConcurrencyThreshold";
+import { withConcurrencyLimit } from "./withConcurrencyLimit";
 
 export class DisassembleXMLFileHandler {
   private readonly ign: Ignore = ignore();
@@ -29,6 +30,7 @@ export class DisassembleXMLFileHandler {
       ignorePath = ".xmldisassemblerignore",
     } = xmlAttributes;
     const resolvedIgnorePath = resolve(ignorePath);
+
     if (existsSync(resolvedIgnorePath)) {
       const content = await readFile(resolvedIgnorePath);
       this.ign.add(content.toString());
@@ -59,48 +61,35 @@ export class DisassembleXMLFileHandler {
       });
     } else if (fileStat.isDirectory()) {
       const subFiles = await readdir(filePath);
-
       const concurrencyLimit = getConcurrencyThreshold();
-      const activePromises: Promise<void>[] = [];
-      let currentIndex = 0;
 
-      // Function to process a single file
-      const processSubFile = async (subFile: string) => {
+      // Create tasks for all subfiles
+      const tasks: (() => Promise<void>)[] = subFiles.map((subFile) => {
         const subFilePath = join(filePath, subFile);
         const relativeSubFilePath = this.posixPath(
           relative(process.cwd(), subFilePath),
         );
 
-        if (
-          subFilePath.endsWith(".xml") &&
-          !this.ign.ignores(relativeSubFilePath)
-        ) {
-          await this.processFile({
-            dirPath: filePath,
-            filePath: subFilePath,
-            uniqueIdElements,
-            prePurge,
-            postPurge,
-          });
-        } else if (this.ign.ignores(relativeSubFilePath)) {
-          logger.warn(`File ignored by ${ignorePath}: ${subFilePath}`);
-        }
-      };
-
-      while (currentIndex < subFiles.length || activePromises.length > 0) {
-        if (
-          currentIndex < subFiles.length &&
-          activePromises.length < concurrencyLimit
-        ) {
-          const subFile = subFiles[currentIndex++];
-          const promise = processSubFile(subFile).finally(() => {
-            activePromises.splice(activePromises.indexOf(promise), 1);
-          });
-          activePromises.push(promise);
-        } else {
-          await Promise.race(activePromises); // Wait for any promise to resolve
-        }
-      }
+        return async () => {
+          if (
+            subFilePath.endsWith(".xml") &&
+            !this.ign.ignores(relativeSubFilePath)
+          ) {
+            await this.processFile({
+              dirPath: filePath,
+              filePath: subFilePath,
+              uniqueIdElements,
+              prePurge,
+              postPurge,
+            });
+          } else if (this.ign.ignores(relativeSubFilePath)) {
+            logger.warn(`File ignored by ${ignorePath}: ${subFilePath}`);
+          }
+        };
+      });
+
+      // Run tasks with concurrency limit
+      await withConcurrencyLimit(tasks, concurrencyLimit);
     }
   }
 

src/service/reassembleXMLFileHandler.ts

@@ -9,6 +9,7 @@ import { buildXMLString } from "@src/service/buildXMLString";
 import { parseXML } from "@src/service/parseXML";
 import { processFilesForRootElement } from "@src/service/processFilesForRootElement";
 import { getConcurrencyThreshold } from "./getConcurrencyThreshold";
+import { withConcurrencyLimit } from "./withConcurrencyLimit";
 
 export class ReassembleXMLFileHandler {
   async processFilesInDirectory(
@@ -27,46 +28,34 @@ export class ReassembleXMLFileHandler {
     let rootResult: [string, string | undefined] | undefined = undefined;
 
     const concurrencyLimit = getConcurrencyThreshold();
-    const activePromises: Promise<void>[] = [];
-    let currentIndex = 0;
-
-    // Function to process a single file
-    const processFile = async (file: string, index: number) => {
-      const filePath = join(dirPath, file);
-      const fileStat = await stat(filePath);
-
-      if (fileStat.isFile() && filePath.endsWith(".xml")) {
-        const xmlParsed = await parseXML(filePath);
-        if (xmlParsed === undefined) return;
-
-        const rootResultFromFile = await processFilesForRootElement(xmlParsed);
-        rootResult = rootResultFromFile;
-
-        const combinedXmlString = buildXMLString(xmlParsed);
-        combinedXmlContents[index] = combinedXmlString;
-      } else if (fileStat.isDirectory()) {
-        const [subCombinedXmlContents, subRootResult] =
-          await this.processFilesInDirectory(filePath);
-        rootResult = subRootResult;
-        combinedXmlContents[index] = subCombinedXmlContents.join("");
-      }
-    };
-
-    while (currentIndex < files.length || activePromises.length > 0) {
-      if (
-        currentIndex < files.length &&
-        activePromises.length < concurrencyLimit
-      ) {
-        const index = currentIndex++;
-        const file = files[index];
-        const promise = processFile(file, index).finally(() => {
-          activePromises.splice(activePromises.indexOf(promise), 1);
-        });
-        activePromises.push(promise);
-      } else {
-        await Promise.race(activePromises); // Wait for any promise to complete
-      }
-    }
+
+    // Create tasks for processing files
+    const tasks: (() => Promise<void>)[] = files.map((file, index) => {
+      return async () => {
+        const filePath = join(dirPath, file);
+        const fileStat = await stat(filePath);
+
+        if (fileStat.isFile() && filePath.endsWith(".xml")) {
+          const xmlParsed = await parseXML(filePath);
+          if (xmlParsed === undefined) return;
+
+          const rootResultFromFile =
+            await processFilesForRootElement(xmlParsed);
+          rootResult = rootResultFromFile;
+
+          const combinedXmlString = buildXMLString(xmlParsed);
+          combinedXmlContents[index] = combinedXmlString;
+        } else if (fileStat.isDirectory()) {
+          const [subCombinedXmlContents, subRootResult] =
+            await this.processFilesInDirectory(filePath);
+          rootResult = subRootResult;
+          combinedXmlContents[index] = subCombinedXmlContents.join("");
+        }
+      };
+    });
+
+    // Execute tasks with concurrency limit
+    await withConcurrencyLimit(tasks, concurrencyLimit);
 
     return [combinedXmlContents.filter(Boolean), rootResult];
   }
@@ -86,6 +75,7 @@ export class ReassembleXMLFileHandler {
       );
       return;
     }
+
     logger.debug(`Parsing directory to reassemble: ${filePath}`);
     const [subCombinedXmlContents, rootResult] =
       await this.processFilesInDirectory(filePath);

src/service/withConcurrencyLimit.ts

@@ -0,0 +1,26 @@
+export async function withConcurrencyLimit<T>(
+  tasks: (() => Promise<T>)[],
+  limit: number,
+): Promise<T[]> {
+  if (limit <= 0) {
+    throw new Error("Concurrency limit must be greater than 0");
+  }
+
+  const results: Promise<T>[] = [];
+  const executing: Promise<T>[] = [];
+
+  for (const task of tasks) {
+    const p = task().then((result) => {
+      executing.splice(executing.indexOf(p), 1);
+      return result;
+    });
+    results.push(p);
+    executing.push(p);
+
+    if (executing.length >= limit) {
+      await Promise.race(executing); // Wait for any promise to resolve
+    }
+  }
+
+  return Promise.all(results);
+}

test/main.spec.ts

@@ -18,6 +18,8 @@ import {
   parseXML,
   buildXMLString,
   XmlElement,
+  getConcurrencyThreshold,
+  withConcurrencyLimit,
 } from "../src/index";
 
 setLogLevel("debug");
@@ -290,6 +292,36 @@ describe("main function", () => {
   it("should compare the files created in the mock directory against the baselines to confirm no changes.", async () => {
     await compareDirectories(baselineDir, mockDir);
   });
+  it("should return a valid concurrency threshold", () => {
+    const threshold = getConcurrencyThreshold();
+    expect(typeof threshold).toBe("number");
+    expect(threshold).toBeGreaterThan(0); // Assuming the threshold must be a positive number.
+  });
+  it("should process tasks with concurrency limit", async () => {
+    const tasks = [
+      () => new Promise((resolve) => setTimeout(() => resolve("Task 1"), 100)),
+      () => new Promise((resolve) => setTimeout(() => resolve("Task 2"), 50)),
+      () => new Promise((resolve) => setTimeout(() => resolve("Task 3"), 10)),
+    ];
+    const results = await withConcurrencyLimit(tasks, 2);
+
+    expect(results).toEqual(["Task 1", "Task 2", "Task 3"]);
+  });
+
+  it("should handle an empty list of tasks", async () => {
+    const results = await withConcurrencyLimit([], 2);
+    expect(results).toEqual([]);
+  });
+
+  it("should throw an error if concurrency limit is invalid", async () => {
+    const tasks = [
+      () => Promise.resolve("Task 1"),
+      () => Promise.resolve("Task 2"),
+    ];
+    await expect(withConcurrencyLimit(tasks, 0)).rejects.toThrow(
+      /Concurrency limit must be greater than 0/,
+    );
+  });
 });
 
 async function compareDirectories(