diff --git a/package.json b/package.json index 17ee7675..cb368598 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,8 @@ "node": ">=14.x" }, "dependencies": { - "@babel/generator": "^7.22.3", - "@babel/parser": "^7.22.3", - "@babel/traverse": "^7.22.1", "@mswjs/interceptors": "^0.19.2", + "@pythagora.io/js-code-processing": "^0.0.4", "axios": "^1.2.2", "blessed": "^0.1.81", "body-parser": "^1.20.1", diff --git a/src/Pythagora.js b/src/Pythagora.js index d243a5a6..2d965823 100644 --- a/src/Pythagora.js +++ b/src/Pythagora.js @@ -9,10 +9,8 @@ const { PYTHAGORA_TESTS_DIR, PYTHAGORA_METADATA_DIR, METADATA_FILENAME, - PYTHAGORA_DELIMITER, - EXPORTED_TESTS_DIR, - EXPORTED_TESTS_DATA_DIR -} = require('./const/common.js'); + PYTHAGORA_DELIMITER +} = require("@pythagora.io/js-code-processing").common; let { BatchInterceptor } = require('@mswjs/interceptors'); let nodeInterceptors = require('@mswjs/interceptors/lib/presets/node.js'); diff --git a/src/RunPythagoraTests.js b/src/RunPythagoraTests.js index 3a9230a1..41b75ef7 100644 --- a/src/RunPythagoraTests.js +++ b/src/RunPythagoraTests.js @@ -1,7 +1,7 @@ const { logTestsFinished, logTestStarting, logTestsStarting } = require('./utils/cmdPrint.js'); const { makeTestRequest } = require('./helpers/testing.js'); const { getCircularReplacer } = require('./utils/common.js') -const { PYTHAGORA_METADATA_DIR, REVIEW_DATA_FILENAME, PYTHAGORA_DELIMITER, PYTHAGORA_TESTS_DIR } = require('./const/common.js'); +const { PYTHAGORA_METADATA_DIR, REVIEW_DATA_FILENAME, PYTHAGORA_DELIMITER, PYTHAGORA_TESTS_DIR } = require("@pythagora.io/js-code-processing").common; const fs = require('fs'); const path = require('path'); diff --git a/src/bin/postinstall.js b/src/bin/postinstall.js index 16bc56f5..65154ab4 100644 --- a/src/bin/postinstall.js +++ b/src/bin/postinstall.js @@ -3,7 +3,7 @@ const fs = require('fs'); const crypto = require('crypto'); const path = require('path'); const { v4: uuidv4 } = require('uuid'); -const {PYTHAGORA_METADATA_DIR, CONFIG_FILENAME, PYTHAGORA_API_SERVER} = require("../const/common"); +const {PYTHAGORA_METADATA_DIR, CONFIG_FILENAME, PYTHAGORA_API_SERVER} = require("@pythagora.io/js-code-processing").common; const packageJson = require('../../package.json'); const pythagoraVersion = packageJson.version; diff --git a/src/bin/run.js b/src/bin/run.js index f0150153..c0c71a7b 100755 --- a/src/bin/run.js +++ b/src/bin/run.js @@ -43,6 +43,7 @@ exec(bashCommand, (error, stdout, stderr) => { const pythagoraRoot = process.argv[1].split('node_modules')[0]; const args = process.argv.slice(2); + // Run the bash script and forward all arguments const child = spawn(bashPath, [bashScript, ...['--pythagora-dir', pythagoraDir, '--pythagora-root', pythagoraRoot, ...args]], { stdio: 'inherit' }); diff --git a/src/commands/export.js b/src/commands/export.js index 17dd49c9..4bd3961b 100644 --- a/src/commands/export.js +++ b/src/commands/export.js @@ -4,7 +4,7 @@ const { EXPORTED_TESTS_DIR, PYTHAGORA_METADATA_DIR, EXPORT_METADATA_FILENAME, -} = require('../const/common'); +} = require("@pythagora.io/js-code-processing").common; const { getAllGeneratedTests } = require("../utils/common"); const { convertOldTestForGPT } = require("../utils/legacy"); const { setUpPythagoraDirs } = require("../helpers/starting"); @@ -12,10 +12,8 @@ const { logAndExit, testEligibleForExportLog, } = require("../utils/cmdPrint"); -const { - isEligibleForExport, - checkForAPIKey -} = require("../helpers/api"); +const {getApiConfig} = require("../helpers/api"); +const {API} = require("@pythagora.io/js-code-processing"); const args = require('../utils/getArgs.js'); const { createDefaultFiles, @@ -25,7 +23,6 @@ const { } = require('../helpers/exports'); async function runExport() { - checkForAPIKey(); setUpPythagoraDirs(); cleanupDataFolder(); let exportsMetadata = JSON.parse(fs.readFileSync(path.resolve(args.pythagora_root, PYTHAGORA_METADATA_DIR, EXPORT_METADATA_FILENAME))); @@ -42,6 +39,9 @@ async function runExport() { await exportTest(originalTest, exportsMetadata); } else { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); + for (let originalTest of generatedTests) { if (originalTest.method === 'OPTIONS') continue; if (testExists(exportsMetadata, originalTest.id)) { @@ -50,7 +50,7 @@ async function runExport() { } let test = convertOldTestForGPT(originalTest); - const isEligible = await isEligibleForExport(test); + const isEligible = await Api.isEligibleForExport(test); if (isEligible) { await exportTest(originalTest, exportsMetadata); diff --git a/src/commands/jest.js b/src/commands/jest.js index c7fc66e0..328055d8 100644 --- a/src/commands/jest.js +++ b/src/commands/jest.js @@ -1,5 +1,5 @@ const jest = require('jest'); -const {EXPORTED_TESTS_DIR, EXPORTED_TESTS_DATA_DIR, SRC_TO_ROOT} = require("../const/common"); +const {EXPORTED_TESTS_DIR, EXPORTED_TESTS_DATA_DIR, SRC_TO_ROOT} = require("@pythagora.io/js-code-processing").common; let fs = require('fs'); const pythagoraJestMethods = require("../helpers/jestMethods"); const {primeJestLog} = require("../utils/cmdPrint"); diff --git a/src/const/common.js b/src/const/common.js deleted file mode 100644 index 973ded95..00000000 --- a/src/const/common.js +++ /dev/null @@ -1,18 +0,0 @@ -module.exports = { - PYTHAGORA_TESTS_DIR: 'pythagora_tests', - PYTHAGORA_METADATA_DIR: '.pythagora', - METADATA_FILENAME: 'metadata.json', - REVIEW_DATA_FILENAME: 'review.json', - EXPORT_METADATA_FILENAME: 'export.json', - CONFIG_FILENAME: 'config.json', - PYTHAGORA_ASYNC_STORE: 42069420, - PYTHAGORA_DELIMITER: '-_-', - EXPORTED_TESTS_DIR: 'pythagora_tests/exported_tests', - EXPORTED_TESTS_DATA_DIR: 'pythagora_tests/exported_tests/data', - SRC_TO_ROOT: '../../../', - MIN_TOKENS_FOR_GPT_RESPONSE: 1640, - MAX_GPT_MODEL_TOKENS: 8192, - PYTHAGORA_UNIT_TESTS_VERSION: 1, - PYTHAGORA_UNIT_DIR: 'pythagora_tests/unit', - PYTHAGORA_API_SERVER: 'https://api.pythagora.io', -} diff --git a/src/helpers/api.js b/src/helpers/api.js index 440864fb..2912d86a 100644 --- a/src/helpers/api.js +++ b/src/helpers/api.js @@ -1,174 +1,12 @@ -const _ = require('lodash'); -const axios = require('axios'); -const { jestAuthFileGenerationLog } = require('../utils/cmdPrint'); -const { bold, reset, red, blue } = require('../utils/cmdPrint').colors; const args = require('../utils/getArgs.js'); -const {PYTHAGORA_UNIT_TESTS_VERSION,PYTHAGORA_API_SERVER} = require('../const/common'); -const API_SERVER = args.pythagora_api_server || PYTHAGORA_API_SERVER; +const {PYTHAGORA_API_SERVER} = require("@pythagora.io/js-code-processing").common; -function extractGPTMessageFromStreamData(input) { - const regex = /data: (.*?)\n/g; - const substrings = []; - let match; - - while ((match = regex.exec(input)) !== null) { - substrings.push(match[1]); +function getApiConfig() { + return { + apiUrl: args.pythagora_api_server || PYTHAGORA_API_SERVER, + apiKey: args.openai_api_key || args.pythagora_api_key, + apiKeyType: args.openai_api_key ? 'openai' : 'pythagora' } - - return substrings.map(s => JSON.parse(s)); -} - -function setOptions({path, method, headers}) { - let apiKey = args.openai_api_key || args.pythagora_api_key; - const parsedUrl = new URL(API_SERVER); - if (!apiKey) throw new Error('No API key provided. Please add --openai-api-key or --pythagora-api-key') - let options = { - protocol: parsedUrl.protocol.replace(':', ''), - hostname: parsedUrl.hostname, - port: parsedUrl.port, - path: path || '/', - method: method || 'POST', - headers: headers || { - 'Content-Type': 'application/json', - 'apikey': apiKey, - 'apikeytype': args.openai_api_key ? 'openai' : 'pythagora' - }, - }; - - if (!options.port) delete options.port; - return options } -async function makeRequest(data, options, customLogFunction) { - let gptResponse = ''; - let httpModule = options.protocol === 'http' ? require('http') : require('https'); - let timeout; - - return new Promise((resolve, reject) => { - const req = httpModule.request(_.omit(options, ['protocol']), function(res){ - res.on('data', (chunk) => { - try { - clearTimeout(timeout); - timeout = setTimeout(() => { - reject(new Error("Request timeout")); - }, 30000); - - let stringified = chunk.toString(); - try { - let json = JSON.parse(stringified); - if (json.error || json.message) { - gptResponse = json; - return; - } - } catch (e) {} - - gptResponse += stringified; - if (gptResponse.indexOf('pythagora_end:') > -1) return; - if (customLogFunction) customLogFunction(gptResponse); - else process.stdout.write(stringified); - } catch (e) {} - }); - res.on('end', async function () { - clearTimeout(timeout); - - process.stdout.write('\n'); - if (res.statusCode >= 400) return reject(new Error(`Response status code: ${res.statusCode}. Error message: ${gptResponse}`)); - if (gptResponse.error) return reject(new Error(`Error: ${gptResponse.error.message}. Code: ${gptResponse.error.code}`)); - if (gptResponse.message) return reject(new Error(`Error: ${gptResponse.message}. Code: ${gptResponse.code}`)); - gptResponse = gptResponse.split('pythagora_end:').pop(); - return resolve(gptResponse); - }); - }); - - req.on('error', (e) => { - clearTimeout(timeout); - console.error("problem with request:"+e.message); - reject(e); - }); - - req.write(data); - - req.end(); - }); -} - -async function getUnitTests(data, customLogFunction) { - let options = setOptions({path: '/api/generate-unit-tests'}); - let tests, error; - try { - tests = await makeRequest(JSON.stringify(data), options, customLogFunction); - } catch (e) { - error = e; - } finally { - return {tests, error}; - } -} - -async function expandUnitTests(data, customLogFunction) { - let options = setOptions({path: '/api/expand-unit-tests'}); - let tests, error; - try { - tests = await makeRequest(JSON.stringify(data), options, customLogFunction); - } catch (e) { - error = e; - } finally { - return {tests, error}; - } -} - -async function getJestAuthFunction(loginMongoQueriesArray, loginRequestBody, loginEndpointPath) { - jestAuthFileGenerationLog(); - - let options = setOptions({path: '/api/generate-jest-auth'}); - return await makeRequest(JSON.stringify({loginMongoQueriesArray, loginRequestBody, loginEndpointPath}), options); -} - - -async function getJestTest(test) { - let options = setOptions({path: '/api/generate-jest-test'}); - return await makeRequest(JSON.stringify(test), options); -} - -async function getJestTestName(test, usedNames) { - let options = setOptions({path:'/api/generate-jest-test-name'}); - return await makeRequest(JSON.stringify({ test }), options); -} - -async function isEligibleForExport(test) { - try { - let options = setOptions({ path: '/api/check-if-eligible' }); - - const response = await axios.post( - `${options.protocol}://${options.hostname}${options.port ? ':' + options.port : ''}${options.path}`, - JSON.stringify({ test }), - { headers: options.headers } - ); - - return response.data; - } catch (error) { - console.log(error); - return false; - } -} - -function checkForAPIKey() { - if (!args.pythagora_api_key && !args.openai_api_key) { - console.log(`${bold+red}No API key found!${reset}`); - console.log('Please run:') - console.log(`${bold+blue}npx pythagora --config --pythagora-api-key ${reset}`); - console.log('or') - console.log(`${bold+blue}npx pythagora --config --openai-api-key ${reset}`); - console.log(`You can get Pythagora API key here: https://mailchi.mp/f4f4d7270a7a/api-waitlist`); - process.exit(0); - } -} - -module.exports = { - getJestAuthFunction, - getJestTest, - getJestTestName, - isEligibleForExport, - getUnitTests, - expandUnitTests, - checkForAPIKey -} +module.exports = {getApiConfig}; diff --git a/src/helpers/exports.js b/src/helpers/exports.js index 5a5d4f89..946bc889 100644 --- a/src/helpers/exports.js +++ b/src/helpers/exports.js @@ -7,7 +7,7 @@ const { METADATA_FILENAME, EXPORT_METADATA_FILENAME, SRC_TO_ROOT -} = require('../const/common'); +} = require("@pythagora.io/js-code-processing").common; const { updateMetadata } = require("../utils/common"); const { convertOldTestForGPT } = require("../utils/legacy"); const { @@ -16,14 +16,10 @@ const { enterLoginRouteLog, testExportStartedLog } = require("../utils/cmdPrint"); -const { - getJestAuthFunction, - getJestTest, - getJestTestName, - cleanupGPTResponse, -} = require("./api"); +const {getApiConfig} = require("./api"); const _ = require('lodash'); const args = require('../utils/getArgs.js'); +const {API} = require("@pythagora.io/js-code-processing"); async function createDefaultFiles(generatedTests) { if (!fs.existsSync('jest.config.js')) { @@ -40,6 +36,9 @@ async function createDefaultFiles(generatedTests) { } async function configureAuthFile(generatedTests) { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); + // TODO make require path better let pythagoraMetadata = require(`../${SRC_TO_ROOT}.pythagora/${METADATA_FILENAME}`); let loginPath = _.get(pythagoraMetadata, 'exportRequirements.login.endpointPath'); @@ -64,7 +63,7 @@ async function configureAuthFile(generatedTests) { } let loginData = pythagoraMetadata.exportRequirements.login; - let code = await getJestAuthFunction(loginData.mongoQueriesArray, loginData.requestBody, loginData.endpointPath); + let code = await Api.getJestAuthFunction(loginData.mongoQueriesArray, loginData.requestBody, loginData.endpointPath); fs.writeFileSync(path.resolve(args.pythagora_root, EXPORTED_TESTS_DIR, 'auth.js'), code); } @@ -97,10 +96,13 @@ function cleanupDataFolder() { } async function exportTest(originalTest, exportsMetadata) { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); + testExportStartedLog(); let test = convertOldTestForGPT(originalTest); - let jestTest = await getJestTest(test); - let testName = await getJestTestName(jestTest, Object.values(exportsMetadata).map(obj => obj.testName)); + let jestTest = await Api.getJestTest(test); + let testName = await Api.getJestTestName(jestTest, Object.values(exportsMetadata).map(obj => obj.testName)); if (!jestTest && !testName) return console.error('There was issue with getting GPT response. Make sure you have access to GPT4 with your API key.'); fs.writeFileSync(`./${EXPORTED_TESTS_DATA_DIR}/${testName.replace('.test.js', '.json')}`, JSON.stringify(test.mongoQueries, null, 2)); diff --git a/src/helpers/jestMethods.js b/src/helpers/jestMethods.js index 735333a3..5c1b17bd 100644 --- a/src/helpers/jestMethods.js +++ b/src/helpers/jestMethods.js @@ -1,6 +1,6 @@ const { cleanupDb, jsonObjToMongo } = require("./mongodb"); const {PYTHAGORA_JEST_DB} = require("../const/mongodb"); -const {EXPORTED_TESTS_DATA_DIR, SRC_TO_ROOT} = require("../const/common"); +const {EXPORTED_TESTS_DATA_DIR, SRC_TO_ROOT} = require("@pythagora.io/js-code-processing").common; const _ = require('lodash'); // TODO can we merge this with the other prepareDB? diff --git a/src/helpers/middlewares.js b/src/helpers/middlewares.js index ff142e78..30bc6843 100644 --- a/src/helpers/middlewares.js +++ b/src/helpers/middlewares.js @@ -3,7 +3,7 @@ const { getCircularReplacer, compareResponse, compareJson, updateMetadata, compa const pythagoraErrors = require("../const/errors"); const { logEndpointNotCaptured, logEndpointCaptured, logWithStoreId } = require("../utils/cmdPrint.js"); const { prepareDB } = require("./mongodb.js"); -const { PYTHAGORA_TESTS_DIR, PYTHAGORA_DELIMITER } = require("../const/common.js"); +const { PYTHAGORA_TESTS_DIR, PYTHAGORA_DELIMITER } = require("@pythagora.io/js-code-processing").common; const bodyParser = require("body-parser"); const {v4} = require("uuid"); diff --git a/src/helpers/mongodb.js b/src/helpers/mongodb.js index 809745c1..73241a33 100644 --- a/src/helpers/mongodb.js +++ b/src/helpers/mongodb.js @@ -16,7 +16,7 @@ const { const {v4} = require("uuid"); const _ = require("lodash"); const { MONGO_METHODS, PYTHAGORA_DB } = require("../const/mongodb"); -const { PYTHAGORA_ASYNC_STORE } = require('../const/common'); +const { PYTHAGORA_ASYNC_STORE } = require("@pythagora.io/js-code-processing").common; let unsupportedMethods = ['aggregate']; // todo remove this methods? diff --git a/src/helpers/mongoose.js b/src/helpers/mongoose.js index 5a01587f..4d6615e2 100644 --- a/src/helpers/mongoose.js +++ b/src/helpers/mongoose.js @@ -1,6 +1,6 @@ const pythagoraErrors = require("../const/errors"); const MODES = require("../const/modes.json"); -const { PYTHAGORA_ASYNC_STORE } = require("../const/common"); +const { PYTHAGORA_ASYNC_STORE } = require("@pythagora.io/js-code-processing").common; const { mongoObjToJson, compareJson, jsonObjToMongo, noUndefined } = require("../utils/common.js"); const { logWithStoreId } = require("../utils/cmdPrint.js"); diff --git a/src/helpers/starting.js b/src/helpers/starting.js index a76aefb0..f206b75c 100644 --- a/src/helpers/starting.js +++ b/src/helpers/starting.js @@ -9,7 +9,7 @@ const { METADATA_FILENAME, EXPORT_METADATA_FILENAME, PYTHAGORA_UNIT_DIR -} = require("../const/common"); +} = require("@pythagora.io/js-code-processing").common; let args = require('../utils/getArgs.js'); diff --git a/src/helpers/unitTests.js b/src/helpers/unitTests.js index c6d94803..9f2b1a93 100644 --- a/src/helpers/unitTests.js +++ b/src/helpers/unitTests.js @@ -1,388 +1,34 @@ const fs = require('fs'); const path = require('path'); -const _ = require('lodash'); -const API = require('./api'); -const {PYTHAGORA_UNIT_DIR} = require("../const/common"); -const generator = require("@babel/generator").default; -const {checkDirectoryExists} = require("../utils/common"); -const { - stripUnrelatedFunctions, - replaceRequirePaths, - getAstFromFilePath, - processAst, - getRelatedFunctions, - getModuleTypeFromFilePath -} = require("../utils/code"); -const {getRelativePath, getFolderTreeItem, getTestFolderPath, checkPathType, isPathInside, calculateDepth} = require("../utils/files"); +const {PYTHAGORA_UNIT_DIR} = require("@pythagora.io/js-code-processing").common; +const {getApiConfig} = require("../helpers/api"); +const { UnitTests, API } = require("@pythagora.io/js-code-processing"); const {initScreenForUnitTests} = require("./cmdGUI"); const {green, red, blue, bold, reset} = require('../utils/cmdPrint').colors; -let functionList = {}, - screen, - scrollableContent, - spinner, - rootPath = '', - queriedPath = '', - folderStructureTree = [], - testsGenerated = [], - skippedFiles = [], - errors = [], - filesToProcess = [], - processedFiles = [], - ignoreFolders = ['node_modules', 'pythagora_tests', '__tests__'], - ignoreFilesEndingWith = [".test.js", ".test.ts", ".test.tsx"], - processExtensions = ['.js', '.ts', '.tsx'], - ignoreErrors = ['BABEL_PARSER_SYNTAX_ERROR'], - force, - isFileToIgnore = (fileName) => { - return ignoreFilesEndingWith.some(ending => fileName.endsWith(ending)) - } -; - -function resolveFilePath(filePath, extension) { - if (fs.existsSync(filePath)) { - return filePath; - } - - const filePathWithExtension = `${filePath}${extension}`; - if (fs.existsSync(filePathWithExtension)) { - return filePathWithExtension; - } - - return undefined; -} - -async function processFile(filePath, filesToProcess) { - try { - let exportsFn = []; - let exportsObj = []; - let functions = []; - let ast = await getAstFromFilePath(filePath); - let syntaxType = await getModuleTypeFromFilePath(ast); - let extension = path.extname(filePath); - - // Analyze dependencies - ast.program.body.forEach(node => { - if (node.type === "ImportDeclaration") { - let importedFile = path.resolve(path.dirname(filePath), node.source.value); - importedFile = resolveFilePath(importedFile, extension); - if (importedFile && !filesToProcess.includes(importedFile)) { - filesToProcess.push(importedFile); - } - } else if (node.type === "VariableDeclaration" && node.declarations.length > 0 && node.declarations[0].init && node.declarations[0].init.type === "CallExpression" && node.declarations[0].init.callee.name === "require") { - let importedFile = path.resolve(path.dirname(filePath), node.declarations[0].init.arguments[0].value); - importedFile = resolveFilePath(importedFile, extension); - if (importedFile && !filesToProcess.includes(importedFile)) { - filesToProcess.push(importedFile); - } - } - }); - - processAst(ast, (funcName, path, type) => { - if (type === 'exportFn' || type === 'exportFnDef') { - exportsFn.push(funcName); - } else if (type === 'exportObj') { - exportsObj.push(funcName); - } - - if (!['exportFn', 'exportObj'].includes(type)) { - functions.push({ - funcName, - code: generator(path.node).code, - filePath: filePath, - relatedFunctions: getRelatedFunctions(path.node, ast, filePath, functionList) - }); - } - }); - for (let f of functions) { - // TODO refactor since this is being set in code.js and here it's reverted - let classParent = exportsFn.find(e => (new RegExp(`${e}\..*`)).test(f.funcName)) || - exportsObj.find(e => (new RegExp(`${e}\..*`)).test(f.funcName)); - - let isExportedAsObject = exportsObj.includes(f.funcName) || exportsObj.includes(classParent); - - // if (classParent) f.funcName = f.funcName.replace(classParent + '.', ''); - - functionList[filePath + ':' + f.funcName] = _.extend(f,{ - classParent, - syntaxType, - exported: exportsFn.includes(f.funcName) || isExportedAsObject || !!classParent, - exportedAsObject: isExportedAsObject, - funcName: f.funcName - }); - } - } catch (e) { - // writeLine(`Error parsing file ${filePath}: ${e}`); - } -} - -async function reformatDataForPythagoraAPI(funcData, filePath, testFilePath) { - let relatedCode = _.groupBy(funcData.relatedCode, 'fileName'); - // TODO add check if there are more functionNames than 1 while exportedAsObject is true - this shouldn't happen ever - relatedCode = _.map(relatedCode, (value, key) => { - return { - fileName: key, - functionNames: value.map(item => item.funcName), - exportedAsObject: value[0].exportedAsObject, - syntaxType: value[0].syntaxType - } - }); - let relatedCodeInSameFile = [funcData.functionName]; - funcData.relatedCode = []; - for (const file of relatedCode) { - if (file.fileName === filePath) { - relatedCodeInSameFile = relatedCodeInSameFile.concat(file.functionNames); - } else { - let fileName = getRelativePath(file.fileName, path.dirname(filePath)); - let code = await stripUnrelatedFunctions(file.fileName, file.functionNames); - let fullPath = filePath.substring(0, filePath.lastIndexOf('/')) + '/' + fileName; - code = replaceRequirePaths(code, filePath, getTestFolderPath(filePath, rootPath)); - funcData.relatedCode.push({ - fileName, - code, - functionNames: file.functionNames, - exportedAsObject: file.exportedAsObject, - syntaxType: file.syntaxType, - pathRelativeToTest: getRelativePath(fullPath, testFilePath) - }); - } - } - funcData.functionCode = await stripUnrelatedFunctions(filePath, relatedCodeInSameFile); - funcData.functionCode = replaceRequirePaths(funcData.functionCode, path.dirname(filePath), getTestFolderPath(filePath, rootPath)); - funcData.pathRelativeToTest = getRelativePath(filePath, testFilePath); - return funcData; -} - -async function createTests(filePath, funcToTest, processingFunction = 'getUnitTests') { - try { - let extension = path.extname(filePath); - let ast = await getAstFromFilePath(filePath); - - const foundFunctions = []; - - processAst(ast, (funcName, path, type) => { - if (type === 'exportFn' || type === 'exportObj') return; - if (funcToTest && funcName !== funcToTest) return; - - let functionFromTheList = functionList[filePath + ':' + funcName]; - if (functionFromTheList && functionFromTheList.exported) { - // TODO refactor since this is being set in code.js and here it's reverted - if (functionFromTheList.classParent) funcName = funcName.replace(functionFromTheList.classParent + '.', '') - foundFunctions.push({ - functionName: funcName, - functionCode: functionFromTheList.code, - relatedCode: functionFromTheList.relatedFunctions, - classParent: functionFromTheList.classParent, - isES6Syntax: functionFromTheList.syntaxType === 'ES6', - exportedAsObject: functionFromTheList.exportedAsObject - }); - } - }); - - const uniqueFoundFunctions = foundFunctions.filter((item, index, self) => - index === self.findIndex((t) => ( - t.functionName === item.functionName && t.functionCode === item.functionCode - )) - ); - - sortFolderTree(folderStructureTree); - - const fileIndex = folderStructureTree.findIndex(item => item.absolutePath === filePath); - for (const [i, funcData] of uniqueFoundFunctions.entries()) { - let indexToPush = fileIndex + 1 + i; - let prefix = folderStructureTree[fileIndex].line.split(path.basename(folderStructureTree[fileIndex].absolutePath))[0]; - folderStructureTree.splice( - indexToPush, - 0, - { - line: " ".repeat(prefix.length) + "└───" + funcData.functionName, - absolutePath: filePath + ':' + funcData.functionName - } - ); - spinner.start(folderStructureTree, indexToPush); - - let testFilePath = path.join(getTestFolderPath(filePath, rootPath), `/${funcData.functionName}.test${extension}`); - if (fs.existsSync(testFilePath) && !force) { - skippedFiles.push(testFilePath); - await spinner.stop(); - folderStructureTree[indexToPush].line = `${green}${folderStructureTree[indexToPush].line}${reset}`; - continue; - } - - let formattedData = await reformatDataForPythagoraAPI(funcData, filePath, getTestFolderPath(filePath, rootPath)); - let { tests, error } = await API[processingFunction](formattedData, (content) => { - scrollableContent.setContent(content); - scrollableContent.setScrollPerc(100); - screen.render(); - }); - - if (tests) { - let testPath = await saveTests(filePath, funcData.functionName, tests); - testsGenerated.push(testPath); - await spinner.stop(); - folderStructureTree[indexToPush].line = `${green}${folderStructureTree[indexToPush].line}${reset}`; - } else if (error) { - errors.push({ - file:filePath, - function: funcData.functionName, - error: { stack: error.stack, message: error.message } - }); - await spinner.stop(); - folderStructureTree[indexToPush].line = `${red}${folderStructureTree[indexToPush].line}${reset}`; - } - } - - if (uniqueFoundFunctions.length > 0) { - folderStructureTree[fileIndex].line = `${green+bold}${folderStructureTree[fileIndex].line}${reset}`; - } - - } catch (e) { - if (!ignoreErrors.includes(e.code)) errors.push(e.stack); - } -} - -async function saveTests(filePath, name, testData) { - let dir = getTestFolderPath(filePath, rootPath); - let extension = path.extname(filePath); - - if (!await checkDirectoryExists(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } +async function generateTestsForDirectory(args) { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); - let testPath = path.join(dir, `/${name}.test${extension}`); - fs.writeFileSync(testPath, testData); - return testPath; -} - -async function traverseDirectory(file, onlyCollectFunctionData, funcName, processingFunction) { - if (processedFiles.includes(file)) { - return; - } - processedFiles.push(file); - - if (await checkPathType(file) === 'file' && !onlyCollectFunctionData) { - if (!processExtensions.includes(path.extname(file))) { - throw new Error('File extension is not supported'); - } - return await createTests(file, funcName, processingFunction); - } - - const absolutePath = path.resolve(file); - const stat = fs.statSync(absolutePath); - - if (!stat.isDirectory() && isFileToIgnore(file)) return; - - if (stat.isDirectory()) { - if (ignoreFolders.includes(path.basename(absolutePath)) || path.basename(absolutePath).charAt(0) === '.') return; - - if (onlyCollectFunctionData && isPathInside(path.dirname(queriedPath), absolutePath)) { - updateFolderTree(absolutePath); - } - - const directoryFiles = fs.readdirSync(absolutePath) - .filter(f => { - const absoluteFilePath = path.join(absolutePath, f); - const fileStat = fs.statSync(absoluteFilePath); - if (fileStat.isDirectory()) { - const baseName = path.basename(absoluteFilePath); - return !ignoreFolders.includes(baseName) && !baseName.startsWith('.'); - } else { - const ext = path.extname(f); - return processExtensions.includes(ext) && !isFileToIgnore(f); - } - }) - .map(f => path.join(absolutePath, f)); - filesToProcess.push(...directoryFiles); - - - } else { - if (!processExtensions.includes(path.extname(absolutePath))) return; - - if (onlyCollectFunctionData) { - if (isPathInside(path.dirname(queriedPath), absolutePath)) { - updateFolderTree(absolutePath); - } - await processFile(absolutePath, filesToProcess); - } else { - await createTests(absolutePath, funcName, processingFunction); - } - } - - while (filesToProcess.length > 0) { - const nextFile = filesToProcess.shift(); - if (processedFiles.includes(nextFile)) { - continue; // Skip processing if it has already been processed - } - await traverseDirectory(nextFile, onlyCollectFunctionData, funcName, processingFunction); - } -} - -function updateFolderTree(absolutePath) { - if (isPathInside(queriedPath, absolutePath) && !folderStructureTree.find(fst => fst.absolutePath === absolutePath)) { - let depth = calculateDepth(queriedPath, absolutePath); - let prefix = ''; - for (let i = 1; i < depth; i++) { - prefix += '| '; - } - folderStructureTree.push(getFolderTreeItem(prefix + "├───", absolutePath)); - } -} - -function sortFolderTree(tree) { - // 1. Sort the folderStructureTree - tree.sort((a, b) => { - if (a.absolutePath < b.absolutePath) { - return -1; - } - if (a.absolutePath > b.absolutePath) { - return 1; - } - return 0; - }); - - // 2. Set prefix according to the position in the directory - for (let i = 0; i < tree.length; i++) { - // Get the current directory path - const currentDirPath = path.dirname(tree[i].absolutePath); - // Check if it's the last file in the directory - if (i === tree.length - 1 || path.dirname(tree[i + 1].absolutePath) !== currentDirPath) { - // Update the prefix for the last file in the directory - tree[i].line = tree[i].line.replace("├───", "└───"); - } - } -} - -async function getFunctionsForExport(path, pythagoraRoot, ignoreFilesRewrite) { - if (ignoreFilesRewrite) { - isFileToIgnore = ignoreFilesRewrite; - } - queriedPath = path; - rootPath = pythagoraRoot; - - await traverseDirectory(queriedPath, true); - processedFiles = []; - await traverseDirectory(queriedPath, true); - processedFiles = []; - return {functionList, folderStructureTree}; -} - -async function generateTestsForDirectory(args, processingFunction = 'getUnitTests') { - let pathToProcess = args.path, - funcName = args.func; - force = args.force; - - API.checkForAPIKey(); - queriedPath = path.resolve(pathToProcess); - rootPath = args.pythagora_root; console.log('Processing folder structure...'); - - await traverseDirectory(queriedPath, true, funcName, processingFunction); - processedFiles = []; - await traverseDirectory(queriedPath, true, funcName, processingFunction); - processedFiles = []; - console.log('Generating tests...'); ({ screen, spinner, scrollableContent } = initScreenForUnitTests()); - await traverseDirectory(queriedPath, false, funcName, processingFunction); + + const unitTests = new UnitTests( + { + pathToProcess: args.path, + pythagoraRoot: args.pythagora_root, + funcName: args.func, + force: args.force + }, + Api, + { + isSaveTests: true, + screen, + spinner, + scrollableContent + } + ); + const {errors, skippedFiles, testsGenerated} = await unitTests.runProcessing(); screen.destroy(); process.stdout.write('\x1B[2J\x1B[0f'); @@ -396,7 +42,7 @@ async function generateTestsForDirectory(args, processingFunction = 'getUnitTest if (testsGenerated.length === 0) { console.log(`${bold+red}No tests generated!${reset}`); } else { - console.log(`Tests are saved in the following directories:${testsGenerated.reduce((acc, item) => acc + '\n' + blue + item, '')}`); + console.log(`Tests are saved in the following directories:${testsGenerated.reduce((acc, item) => acc + '\n' + blue + item.testPath, '')}`); console.log(`${bold+green}${testsGenerated.length} unit tests generated!${reset}`); } @@ -404,7 +50,5 @@ async function generateTestsForDirectory(args, processingFunction = 'getUnitTest } module.exports = { - getFunctionsForExport, - generateTestsForDirectory, - sortFolderTree + generateTestsForDirectory } diff --git a/src/helpers/unitTestsExpand.js b/src/helpers/unitTestsExpand.js index 5532406c..78b1106f 100644 --- a/src/helpers/unitTestsExpand.js +++ b/src/helpers/unitTestsExpand.js @@ -1,253 +1,53 @@ -const fs = require("fs"); -const path = require("path"); -const _ = require("lodash"); -const { expandUnitTests, checkForAPIKey } = require("./api"); -const { - PYTHAGORA_UNIT_DIR, -} = require("../const/common"); -const { checkDirectoryExists } = require("../utils/common"); -const { - getAstFromFilePath, - getRelatedTestImports, - getSourceCodeFromAst, - getModuleTypeFromFilePath, - replaceRequirePaths -} = require("../utils/code"); -const { getFunctionsForExport, sortFolderTree } = require("./unitTests"); -const { checkPathType, getRelativePath, getTestFolderPath } = require("../utils/files"); -const { initScreenForUnitTests } = require("./cmdGUI"); -const { green, red, blue, bold, reset } = require("../utils/cmdPrint").colors; +const fs = require('fs'); +const path = require('path'); +const {PYTHAGORA_UNIT_DIR} = require("@pythagora.io/js-code-processing").common; +const {getApiConfig} = require("../helpers/api"); +const { UnitTestsExpand, API } = require("@pythagora.io/js-code-processing"); +const {initScreenForUnitTests} = require("./cmdGUI"); +const {green, red, blue, bold, reset} = require('../utils/cmdPrint').colors; -let functionList = {}, - screen, - scrollableContent, - spinner, - rootPath = "", - queriedPath = "", - folderStructureTree = [], - testsGenerated = [], - skippedFiles = [], - errors = [], - ignoreFolders = ["node_modules", "pythagora_tests"], - filesEndingWith = [".js", ".ts", ".tsx"], - processExtensions = [".js", ".ts"], - ignoreErrors = ["BABEL_PARSER_SYNTAX_ERROR"], - force; - -async function saveTests(filePath, fileName, newTests) { - let dir = filePath.substring(0, filePath.lastIndexOf("/")); - if (!await checkDirectoryExists(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - let testPath = path.join(dir, `/${fileName}`); - fs.writeFileSync(testPath, newTests); - return testPath; -} - -function reformatDataForPythagoraAPI(filePath, testCode, relatedCode, syntaxType) { - const importedFiles = []; - _.forEach(relatedCode, (f) => { - const testPath = path.join( - path.resolve(PYTHAGORA_UNIT_DIR), - filePath.replace(rootPath, "") - ); - const pathRelativeToTest = getRelativePath(f.filePath, testPath.substring(0, testPath.lastIndexOf("/"))); - f.pathRelativeToTest = pathRelativeToTest; - if (!importedFiles.find(i => i.filePath == f.filePath)) { - importedFiles.push({ - fileName: f.fileName.substring(f.fileName.lastIndexOf("/") + 1), - filePath: f.filePath, - pathRelativeToTest: f.pathRelativeToTest, - syntaxType: f.syntaxType - }); - } - if (f.relatedFunctions.length) { - f.relatedFunctions = _.map(f.relatedFunctions, (f) => ({...f, fileName: f.fileName.substring(f.fileName.lastIndexOf("/") + 1)}) ) - f.relatedFunctions.forEach((f) => importedFiles.push({ - ...f, - pathRelativeToTest: getRelativePath(f.filePath, testPath.substring(0, testPath.lastIndexOf("/"))) - })) - } - }) - const testFilePath = getTestFolderPath(filePath, rootPath); - const pathRelativeToTest = getRelativePath(filePath, testFilePath); - return { - testFileName: filePath.substring(filePath.lastIndexOf("/") + 1), - testCode, - relatedCode, - importedFiles, - isES6Syntax: syntaxType === "ES6", - pathRelativeToTest, - filePath - }; -} - -async function createAdditionalTests(filePath, prefix) { - try { - const ast = await getAstFromFilePath(filePath); - let syntaxType = await getModuleTypeFromFilePath(ast); - const testPath = path.join( - path.resolve(PYTHAGORA_UNIT_DIR), - filePath.replace(rootPath, "") - ); - - let testCode = getSourceCodeFromAst(ast); - testCode = replaceRequirePaths( - testCode, - path.dirname(filePath), - testPath.substring(0, testPath.lastIndexOf("/")) - ); - - sortFolderTree(folderStructureTree); - - const relatedCode = getRelatedTestImports(ast, filePath, functionList); - const formattedData = reformatDataForPythagoraAPI( - filePath, - testCode, - relatedCode, - syntaxType - ); - - const fileIndex = folderStructureTree.findIndex( - (item) => item.absolutePath === filePath - ); - spinner.start(folderStructureTree, fileIndex); - - if (fs.existsSync(testPath) && !force) { - skippedFiles.push(testPath); - await spinner.stop(); - folderStructureTree[ - fileIndex - ].line = `${green}${folderStructureTree[fileIndex].line}${reset}`; - return; - } - - let { tests, error } = await expandUnitTests(formattedData, (content) => { - scrollableContent.setContent(content); - scrollableContent.setScrollPerc(100); - screen.render(); - }); - - if (tests) { - await saveTests(testPath, formattedData.testFileName, tests); - testsGenerated.push(testPath); - await spinner.stop(); - folderStructureTree[ - fileIndex - ].line = `${green}${folderStructureTree[fileIndex].line}${reset}`; - } else if (error) { - errors.push({ - file: filePath, - error: { stack: error.stack, message: error.message }, - }); - await spinner.stop(); - folderStructureTree[ - fileIndex - ].line = `${red}${folderStructureTree[fileIndex].line}${reset}`; +async function expandTestsForDirectory(args) { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); + + console.log('Processing folder structure...'); + ({ screen, spinner, scrollableContent } = initScreenForUnitTests()); + + const unitTestsExpand = new UnitTestsExpand( + { + pathToProcess: args.path, + pythagoraRoot: args.pythagora_root, + force: args.force + }, + Api, + { + isSaveTests: true, + screen, + spinner, + scrollableContent + } + ); + const {errors, skippedFiles, testsGenerated} = await unitTestsExpand.runProcessing(); + + screen.destroy(); + process.stdout.write('\x1B[2J\x1B[0f'); + if (errors.length) { + let errLogPath = `${path.resolve(PYTHAGORA_UNIT_DIR, 'errorLogs.log')}`; + fs.writeFileSync(errLogPath, JSON.stringify(errors, null, 2)); + console.error('There were errors encountered while trying to expand unit tests.\n'); + console.error(`You can find logs here: ${errLogPath}`); } - } catch (e) { - if (!ignoreErrors.includes(e.code)) errors.push(e.stack); - } -} - -function checkForTestFilePath(filePath) { - const pattern = /test\.(js|ts|tsx)$/; - return pattern.test(filePath); -} - -async function traverseDirectoryTests(directory, prefix = "") { - if ( - (await checkPathType(directory)) === "file" && - checkForTestFilePath(directory) - ) { - const newPrefix = `| ${prefix}| `; - return await createAdditionalTests(directory, newPrefix); - } else if ( - (await checkPathType(directory)) === "file" && - !checkForTestFilePath(directory) - ) { - throw new Error("Invalid test file path"); - } - - const files = fs.readdirSync(directory); - for (const file of files) { - const absolutePath = path.join(directory, file); - const stat = fs.statSync(absolutePath); - if (stat.isDirectory()) { - if ( - ignoreFolders.includes(path.basename(absolutePath)) || - path.basename(absolutePath).charAt(0) === "." - ) - continue; - await traverseDirectoryTests(absolutePath, prefix); + if (skippedFiles.length) console.log(`${bold}Generation of ${skippedFiles.length} test suites were skipped because tests already exist. If you want to override them add "--force" flag to command${reset}`); + if (testsGenerated.length === 0) { + console.log(`${bold+red}No tests generated!${reset}`); } else { - if ( - !processExtensions.includes(path.extname(absolutePath)) || - !checkForTestFilePath(file) - ) - continue; - await createAdditionalTests(absolutePath, prefix); - } - } -} - -async function expandTestsForDirectory(args) { - let pathToProcess = args.path; - force = args.force; - - checkForAPIKey(); - queriedPath = path.resolve(pathToProcess); - rootPath = args.pythagora_root; - console.log("Processing folder structure..."); - - const exportData = await getFunctionsForExport( - queriedPath, - rootPath, - (fileName) => { - return !filesEndingWith.some((ending) => fileName.endsWith(ending)); + console.log(`Tests are saved in the following directories:${testsGenerated.reduce((acc, item) => acc + '\n' + blue + item.testPath, '')}`); + console.log(`${bold+green}${testsGenerated.length} unit tests generated!${reset}`); } - ); - functionList = exportData.functionList; - folderStructureTree = exportData.folderStructureTree; - folderStructureTree = folderStructureTree.filter( - (i) => checkForTestFilePath(i.absolutePath) || i.isDirectory - ); - - ({ screen, spinner, scrollableContent } = initScreenForUnitTests()); - - await traverseDirectoryTests(queriedPath, false); - - screen.destroy(); - process.stdout.write("\x1B[2J\x1B[0f"); - if (errors.length) { - let errLogPath = `${path.resolve(PYTHAGORA_UNIT_DIR, "errorLogs.log")}`; - fs.writeFileSync(errLogPath, JSON.stringify(errors, null, 2)); - console.error( - "There were errors encountered while trying to expand unit tests.\n" - ); - console.error(`You can find logs here: ${errLogPath}`); - } - if (skippedFiles.length) - console.log( - `${bold}Generation of ${skippedFiles.length} test suites were skipped because tests already exist. If you want to override them add "--force" flag to command${reset}` - ); - if (testsGenerated.length === 0) { - console.log(`${bold + red}No tests generated!${reset}`); - } else { - console.log( - `Tests are saved in the following directories:${testsGenerated.reduce( - (acc, item) => acc + "\n" + blue + item, - "" - )}` - ); - console.log( - `${bold + green}${testsGenerated.length} unit tests generated!${reset}` - ); - } - - process.exit(0); + + process.exit(0); } module.exports = { - expandTestsForDirectory, + expandTestsForDirectory, }; diff --git a/src/scripts/deleteAllFailed.js b/src/scripts/deleteAllFailed.js index 2b7119cd..c790dc18 100644 --- a/src/scripts/deleteAllFailed.js +++ b/src/scripts/deleteAllFailed.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { REVIEW_DATA_FILENAME, PYTHAGORA_METADATA_DIR, PYTHAGORA_TESTS_DIR, METADATA_FILENAME, PYTHAGORA_DELIMITER } = require('../const/common.js'); +const { REVIEW_DATA_FILENAME, PYTHAGORA_METADATA_DIR, PYTHAGORA_TESTS_DIR, METADATA_FILENAME, PYTHAGORA_DELIMITER } = require("@pythagora.io/js-code-processing").common; const { getCircularReplacer } = require('../utils/common.js'); const { logAndExit } = require('../utils/cmdPrint.js'); let args = require('../utils/getArgs.js'); diff --git a/src/scripts/deleteTest.js b/src/scripts/deleteTest.js index 9d46e230..9a9a04b8 100644 --- a/src/scripts/deleteTest.js +++ b/src/scripts/deleteTest.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { PYTHAGORA_TESTS_DIR, PYTHAGORA_DELIMITER } = require('../const/common.js'); +const { PYTHAGORA_TESTS_DIR, PYTHAGORA_DELIMITER } = require("@pythagora.io/js-code-processing").common; const { getCircularReplacer } = require('../utils/common.js'); const { logAndExit } = require('../utils/cmdPrint.js'); const args = require('../utils/getArgs.js'); diff --git a/src/scripts/enterData.js b/src/scripts/enterData.js index c6354266..c3f1b82f 100644 --- a/src/scripts/enterData.js +++ b/src/scripts/enterData.js @@ -1,7 +1,7 @@ const {loginRouteEnteredLog} = require("../utils/cmdPrint"); const {updateMetadata} = require("../utils/common"); const {setUpPythagoraDirs} = require("../helpers/starting"); -const {METADATA_FILENAME, SRC_TO_ROOT} = require("../const/common"); +const {METADATA_FILENAME, SRC_TO_ROOT} = require("@pythagora.io/js-code-processing").common; // TODO make require path better!! const readline = require("readline"); const _ = require("lodash"); diff --git a/src/scripts/review.js b/src/scripts/review.js index 54f679e1..6c97566e 100644 --- a/src/scripts/review.js +++ b/src/scripts/review.js @@ -4,7 +4,7 @@ const readline = require('readline'); const _ = require('lodash'); let args = require('../utils/getArgs.js'); -const { PYTHAGORA_TESTS_DIR, PYTHAGORA_METADATA_DIR, REVIEW_DATA_FILENAME } = require('../const/common.js'); +const { PYTHAGORA_TESTS_DIR, PYTHAGORA_METADATA_DIR, REVIEW_DATA_FILENAME } = require("@pythagora.io/js-code-processing").common; const { logChange, logAndExit } = require('../utils/cmdPrint.js'); const { compareJson, getMetadata } = require('../utils/common.js'); diff --git a/src/scripts/testsEligibleForExport.js b/src/scripts/testsEligibleForExport.js index af44809d..e4a4552f 100644 --- a/src/scripts/testsEligibleForExport.js +++ b/src/scripts/testsEligibleForExport.js @@ -3,17 +3,21 @@ const path = require("path"); const {getAllGeneratedTests} = require("../utils/common"); const {convertOldTestForGPT} = require("../utils/legacy"); const {testEligibleForExportLog} = require("../utils/cmdPrint"); -const {isEligibleForExport} = require("../helpers/api"); +const {getApiConfig} = require("../helpers/api"); +const {API} = require("@pythagora.io/js-code-processing"); const args = require("../utils/getArgs"); -const {getFunctionsForExport} = require("../helpers/unitTests"); +const {UnitTestsCommon} = require("@pythagora.io/js-code-processing"); async function testsEligibleForExport() { + const { apiUrl, apiKey, apiKeyType } = getApiConfig(); + const Api = new API(apiUrl, apiKey, apiKeyType); + let csvData = 'endpoint,testId,tokens\n'; let tests = getAllGeneratedTests(); for (let test of tests) { test = convertOldTestForGPT(test); - let isEligible = await isEligibleForExport(test); + let isEligible = await Api.isEligibleForExport(test); testEligibleForExportLog(test.endpoint, test.testId, isEligible); csvData += `${test.endpoint},${test.testId},${isEligible ? 'TRUE' : 'FALSE'}\n`; } @@ -25,7 +29,13 @@ async function testsEligibleForExport() { } async function unitTestsEligibleForExport() { - let tests = await getFunctionsForExport(args.dir); + const unitTestsCommon = new UnitTestsCommon({ + pathToProcess: args.path, + pythagoraRoot: args.pythagora_root + }); + await unitTestsCommon.traverseAllDirectories(); + + let tests = unitTestsCommon.functionList; let csvData = 'fileName,functionName,relatedFunctions\n'; for (let path in tests) { let funcName = path.substring(path.lastIndexOf(':') + 1); diff --git a/src/utils/cmdPrint.js b/src/utils/cmdPrint.js index 92aa47c4..31698e84 100644 --- a/src/utils/cmdPrint.js +++ b/src/utils/cmdPrint.js @@ -1,14 +1,8 @@ let { cutWithDots, compareJson, compareJsonDetailed } = require('./common'); let pythagoraErrors = require('../const/errors'); let args = require('../utils/getArgs.js'); -let { PYTHAGORA_DELIMITER } = require('../const/common'); - -let red = '\x1b[31m', - yellow = '\x1b[33m', - green = '\x1b[32m', - blue = '\x1b[34m', - reset = '\x1b[0m', - bold = '\x1b[1m'; +let { PYTHAGORA_DELIMITER } = require("@pythagora.io/js-code-processing").common; +const {red, yellow, green, blue, reset, bold } = require("@pythagora.io/js-code-processing").colors; let logWithStoreId = (msg) => { const id = global.asyncLocalStorage.getStore(); diff --git a/src/utils/code.js b/src/utils/code.js deleted file mode 100644 index c392bad1..00000000 --- a/src/utils/code.js +++ /dev/null @@ -1,426 +0,0 @@ -const path = require("path"); -const babelParser = require("@babel/parser"); -const {default: babelTraverse} = require("@babel/traverse"); -const {default: generator} = require("@babel/generator"); -const {getRelativePath} = require("./files"); -const fs = require("fs").promises; -const _ = require("lodash"); - - -function replaceRequirePaths(code, currentPath, testFilePath) { - const importRequirePathRegex = /(require\((['"`])(.+?)\2\))|(import\s+.*?\s+from\s+(['"`])(.+?)\5)/g; - - return code.replace(importRequirePathRegex, (match, requireExp, requireQuote, requirePath, importExp, importQuote, importPath) => { - let quote, modulePath; - - if (requireExp) { - quote = requireQuote; - modulePath = requirePath; - } else if (importExp) { - quote = importQuote; - modulePath = importPath; - } - - if (!modulePath.startsWith('./') && !modulePath.startsWith('../')) return match; - - const absoluteRequirePath = path.resolve(currentPath, modulePath); - - const newRequirePath = getRelativePath(absoluteRequirePath, testFilePath); - - if (requireExp) { - return `require(${quote}${newRequirePath}${quote})`; - } else if (importExp) { - return `${importExp.split('from')[0].trim()} from ${quote}${newRequirePath}${quote}`; - } - }); -} - -async function getAstFromFilePath(filePath) { - let data = await fs.readFile(filePath, 'utf8'); - let nodeTypesStack = []; - // Remove shebang if it exists - if (data.indexOf('#!') === 0) { - data = '//' + data; - } - - const ast = babelParser.parse(data, { - sourceType: "module", // Consider input as ECMAScript module - locations: true, - plugins: ["jsx", "objectRestSpread", "typescript"] // Enable JSX, typescript and object rest/spread syntax - }); - - return ast; -} - -async function getModuleTypeFromFilePath(ast) { - let moduleType = 'CommonJS'; - - babelTraverse(ast, { - ImportDeclaration(path) { - moduleType = 'ES6'; - path.stop(); // Stop traversal when an ESM statement is found - }, - ExportNamedDeclaration(path) { - moduleType = 'ES6'; - path.stop(); // Stop traversal when an ESM statement is found - }, - ExportDefaultDeclaration(path) { - moduleType = 'ES6'; - path.stop(); // Stop traversal when an ESM statement is found - }, - CallExpression(path) { - if (path.node.callee.name === 'require') { - moduleType = 'CommonJS'; - path.stop(); // Stop traversal when a CommonJS statement is found - } - }, - AssignmentExpression(path) { - if (path.node.left.type === 'MemberExpression' && path.node.left.object.name === 'module' && path.node.left.property.name === 'exports') { - moduleType = 'CommonJS'; - path.stop(); // Stop traversal when a CommonJS statement is found - } - } - }); - - return moduleType; -} - -function collectTopRequires(node) { - let requires = []; - babelTraverse(node, { - VariableDeclaration(path) { - if (path.node.declarations[0].init && path.node.declarations[0].init.callee && path.node.declarations[0].init.callee.name === 'require') { - requires.push(generator(path.node).code); - } - }, - ImportDeclaration(path) { - requires.push(generator(path.node).code); - } - }); - return requires; -} - -function insideFunctionOrMethod(nodeTypesStack) { - return nodeTypesStack.slice(0, -1).some(type => /^(FunctionDeclaration|FunctionExpression|ArrowFunctionExpression|ClassMethod)$/.test(type)); -} - -function getPathFromRequireOrImport(path) { - return (path.match(/require\((['"`])(.*?)\1\)|import\s+.*?\s+from\s+(['"`])(.*?)\3/) || [])[2] || - (path.match(/require\((['"`])(.*?)\1\)|import\s+.*?\s+from\s+(['"`])(.*?)\3/) || [])[4]; -} - -function getFullPathFromRequireOrImport(importPath, filePath) { - if (importPath && (importPath.startsWith('./') || importPath.startsWith('../'))) importPath = path.resolve(filePath.substring(0, filePath.lastIndexOf('/')), importPath); - if (importPath.lastIndexOf('.js') + '.js'.length !== importPath.length) importPath += '.js'; - return importPath; -} - -function getRelatedFunctions(node, ast, filePath, functionList) { - let relatedFunctions = []; - let requiresFromFile = collectTopRequires(ast); - - function processNodeRecursively(node) { - if (node.type === 'CallExpression') { - let funcName; - let callee = node.callee; - - while (callee.type === 'MemberExpression') { - callee = callee.object; - } - - if (callee.type === 'Identifier') { - funcName = callee.name; - } else if (callee.type === 'MemberExpression') { - funcName = callee.property.name; - if (callee.object.type === 'Identifier') { - funcName = callee.object.name + '.' + funcName; - } - } - - let requiredPath = requiresFromFile.find(require => require.includes(funcName)); - const importPath = requiredPath; - if (!requiredPath) { - requiredPath = filePath; - } else { - requiredPath = getPathFromRequireOrImport(requiredPath); - requiredPath = getFullPathFromRequireOrImport(requiredPath, filePath); - } - let functionFromList = functionList[requiredPath + ':' + funcName]; - if (functionFromList) { - relatedFunctions.push(_.extend(functionFromList, { - fileName: requiredPath, - importPath - })); - } - } - - // Traverse child nodes - for (const key in node) { - const prop = node[key]; - if (Array.isArray(prop)) { - for (const child of prop) { - if (typeof child === 'object' && child !== null) { - processNodeRecursively(child); - } - } - } else if (typeof prop === 'object' && prop !== null) { - processNodeRecursively(prop); - } - } - } - - processNodeRecursively(node); - return relatedFunctions; -} - -async function stripUnrelatedFunctions(filePath, targetFuncNames) { - const ast = await getAstFromFilePath(filePath); - - // Store the node paths of unrelated functions and class methods - const unrelatedNodes = []; - - processAst(ast, (funcName, path, type) => { - if (!targetFuncNames.includes(funcName) && type !== 'exportFn' && type !== 'exportObj') { - // If the function is being used as a property value, remove the property instead of the function - if (path.parentPath.isObjectProperty()) { - unrelatedNodes.push(path.parentPath); - } else { - unrelatedNodes.push(path); - } - } - }); - - // Remove unrelated nodes from the AST - for (const path of unrelatedNodes) { - path.remove(); - } - - // Generate the stripped code from the modified AST - const strippedCode = generator(ast).code; - - return strippedCode; -} - -function processAst(ast, cb) { - let nodeTypesStack = []; - babelTraverse(ast, { - enter(path) { - nodeTypesStack.push(path.node.type); - if (insideFunctionOrMethod(nodeTypesStack)) return; - - // Handle module.exports - if (path.isExpressionStatement()) { - const expression = path.node.expression; - if (expression && expression.type === 'AssignmentExpression') { - const left = expression.left; - if (left.object && left.object.type === 'MemberExpression' && - left.object.object.name === 'module' && - left.object.property.name === 'exports') { - if (expression.right.type === 'Identifier') { - // module.exports.func1 = func1 - return cb(left.property.name, path, 'exportObj'); - } else if (expression.right.type === 'FunctionExpression') { - // module.exports.funcName = function() { ... } - // module.exports = function() { ... } - const loc = path.node.loc.start; - let funcName = (left.property.name) || `anon_func_${loc.line}_${loc.column}`; - return cb(funcName, path, 'exportObj'); - } - } else if (left.type === 'MemberExpression' && - left.object.name === 'module' && - left.property.name === 'exports') { - if (expression.right.type === 'Identifier') { - // module.exports = func1 - return cb(expression.right.name, path, 'exportFn'); - } else if (expression.right.type === 'FunctionExpression') { - let funcName; - if (expression.right.id) { - // module.exports = function func1() { ... } - funcName = expression.right.id.name; - } else { - // module.exports = function() { ... } - const loc = path.node.loc.start; - funcName = `anon_func_${loc.line}_${loc.column}`; - } - return cb(funcName, path, 'exportFnDef'); - } else if (expression.right.type === 'ObjectExpression') { - expression.right.properties.forEach(prop => { - if (prop.type === 'ObjectProperty') { - // module.exports = { func1 }; - return cb(prop.key.name, path, 'exportObj'); - } - }); - } - } - // Handle TypeScript transpiled exports - else if (left.type === 'MemberExpression' && - left.object.name === 'exports') { - // exports.func1 = function() { ... } - // exports.func1 = func1 - return cb(left.property.name, path, 'exportObj'); - } - } - } - - // Handle ES6 export statements - if (path.isExportDefaultDeclaration()) { - const declaration = path.node.declaration; - if (declaration.type === 'FunctionDeclaration' || declaration.type === 'Identifier') { - // export default func1; - // TODO export default function() { ... } - // TODO cover anonimous functions - add "anon_" name - return cb(declaration.id ? declaration.id.name : declaration.name, path, 'exportFn'); - } else if (declaration.type === 'ObjectExpression') { - declaration.properties.forEach(prop => { - if (prop.type === 'ObjectProperty') { - // export default { func1: func } - // export default { func1 } - return cb(prop.key.name, path, 'exportObj'); - } - }); - } else if (declaration.type === 'ClassDeclaration') { - // export default class Class1 { ... } - return cb(declaration.id ? declaration.id.name : declaration.name, path, 'exportFnDef'); - } - } else if (path.isExportNamedDeclaration()) { - if (path.node.declaration) { - if (path.node.declaration.type === 'FunctionDeclaration') { - // export function func1 () { ... } - // export class Class1 () { ... } - return cb(path.node.declaration.id.name, path, 'exportObj'); - } else if (path.node.declaration.type === 'VariableDeclaration') { - // export const const1 = 'constant'; - // export const func1 = () => { ... } - path.node.declaration.declarations.forEach(declaration => { - return cb(declaration.id.name, path, 'exportObj'); - }); - } else if (path.node.declaration.type === 'ClassDeclaration') { - // export class Class1 { ... } - return cb(path.node.declaration.id.name, path, 'exportFnDef'); - } - } else if (path.node.specifiers.length > 0) { - path.node.specifiers.forEach(spec => { - // export { func as func1 } - return cb(spec.exported.name, path, 'exportObj'); - }); - } - } - - let funcName; - if (path.isFunctionDeclaration()) { - funcName = path.node.id.name; - } else if (path.isFunctionExpression() || path.isArrowFunctionExpression()) { - if (path.parentPath.isVariableDeclarator()) { - funcName = path.parentPath.node.id.name; - } else if (path.parentPath.isAssignmentExpression() || path.parentPath.isObjectProperty()) { - funcName = path.parentPath.node.left ? path.parentPath.node.left.name : path.parentPath.node.key.name; - } - } else if (path.node.type === 'ClassMethod' && path.node.key.name !== 'constructor') { - funcName = path.node.key.name; - if (path.parentPath.node.type === 'ClassDeclaration') { - const className = path.parentPath.node.id.name; - funcName = `${className}.${funcName}`; - } else if (path.parentPath.node.type === 'ClassExpression') { - const className = path.parentPath.node.id.name || ''; - funcName = `${className}.${funcName}`; - } else if (path.parentPath.node.type === 'ClassBody') { - // TODO: Handle classes that are not declared as a variable - const className = path.parentPath.parentPath.node.id ? path.parentPath.parentPath.node.id.name : ''; - funcName = `${className}.${funcName}`; - } - } - - if (funcName) cb(funcName, path); - }, - exit(path) { - nodeTypesStack.pop(); - } - }); -} - -function getSourceCodeFromAst (ast) { - return generator(ast).code; -} - -function collectTestRequires(node) { - let requires = []; - babelTraverse(node, { - ImportDeclaration(path) { - if (path.node && path.node.specifiers && path.node.specifiers.length > 0) { - const requireData = { - code: generator(path.node).code, - functionNames: [] - } - - _.forEach(path.node.specifiers, (s) => { - if (s.local && s.local.name) requireData.functionNames.push(s.local.name) - }) - - requires.push(requireData); - } - }, - CallExpression(path) { - if (path.node.callee.name === 'require' && path.node.arguments && path.node.arguments.length > 0) { - const requireData = { - code: generator(path.node).code, - functionNames: [] - } - - // In case of a CommonJS require, the function name is usually the variable identifier of the parent node - if (path.parentPath && path.parentPath.node.type === 'VariableDeclarator' && path.parentPath.node.id) { - requireData.functionNames.push(path.parentPath.node.id.name) - } - - requires.push(requireData); - } - } - }); - return requires; -} - -function getRelatedTestImports(ast, filePath, functionList) { - let relatedCode = []; - let requiresFromFile = collectTestRequires(ast); - - for (let fileImport in requiresFromFile) { - let requiredPath = getPathFromRequireOrImport(requiresFromFile[fileImport].code); - requiredPath = getFullPathFromRequireOrImport(requiredPath, filePath); - - _.forEach(requiresFromFile[fileImport].functionNames, (funcName) => { - let functionFromList = functionList[requiredPath + ':' + funcName]; - if (functionFromList) { - relatedCode.push(_.extend(functionFromList, { - fileName: requiredPath - })); - } - }) - } - - for (let relCode of relatedCode) { - let relatedCodeImports = ''; - for (let func of relCode.relatedFunctions) { - if (func.importPath) { - relatedCodeImports += `${func.importPath}\n`; - } - } - - if (relatedCodeImports) { - relCode.code = `${relatedCodeImports}\n${relCode.code}`; - } - } - - return relatedCode; -} - -module.exports = { - replaceRequirePaths, - getAstFromFilePath, - collectTopRequires, - insideFunctionOrMethod, - getRelatedFunctions, - stripUnrelatedFunctions, - processAst, - getModuleTypeFromFilePath, - getSourceCodeFromAst, - getRelatedTestImports -} diff --git a/src/utils/common.js b/src/utils/common.js index a8c062d2..72a67234 100644 --- a/src/utils/common.js +++ b/src/utils/common.js @@ -6,7 +6,7 @@ const { METADATA_FILENAME, PYTHAGORA_TESTS_DIR, PYTHAGORA_DELIMITER -} = require("../const/common"); +} = require("@pythagora.io/js-code-processing").common; const path = require('path'); const args = require('./getArgs.js'); let mongodb; @@ -354,22 +354,7 @@ function comparePaths(path1, path2) { return path1 === path2; } -async function checkDirectoryExists(directoryPath) { - try { - const stats = await fs.promises.stat(directoryPath); - return stats.isDirectory(); - } catch (error) { - if (error.code === 'ENOENT') { - // Directory does not exist - return false; - } - // Other error occurred - throw error; - } -} - module.exports = { - checkDirectoryExists, compareJson, compareJsonDetailed, comparePaths, diff --git a/src/utils/files.js b/src/utils/files.js deleted file mode 100644 index 2d49397d..00000000 --- a/src/utils/files.js +++ /dev/null @@ -1,57 +0,0 @@ -const path = require("path"); -const {PYTHAGORA_UNIT_DIR} = require("../const/common"); -const fs = require("fs").promises; -const fsSync = require("fs"); - - -async function checkPathType(path) { - let stats = await fs.stat(path); - return stats.isFile() ? 'file' : 'directory'; -} - -function getRelativePath(filePath, referenceFolderPath) { - let relativePath = path.relative(path.resolve(referenceFolderPath), filePath); - if (!relativePath.startsWith('../') && !relativePath.startsWith('./')) { - relativePath = './' + relativePath; - } - return relativePath; -} - -function getFolderTreeItem(prefix, absolutePath) { - const isDirectory = absolutePath.includes(':') ? false : fsSync.statSync(absolutePath).isDirectory(); - return { - line: `${prefix}${path.basename(absolutePath)}`, - absolutePath, - isDirectory - }; -} - -function isPathInside(basePath, targetPath) { - const relativePath = path.relative(basePath, targetPath); - return !relativePath || (!relativePath.startsWith('..') && !path.isAbsolute(relativePath)); -} - -function getTestFolderPath(filePath, rootPath) { - return path.join( - path.resolve(PYTHAGORA_UNIT_DIR), - path.dirname(filePath).replace(path.resolve(rootPath), ''), - path.basename(filePath, path.extname(filePath)) - ); -} - -function calculateDepth(basePath, targetPath) { - const baseComponents = basePath.split(path.sep); - const targetComponents = targetPath.split(path.sep); - - // The depth is the difference in the number of components - return targetComponents.length - baseComponents.length + 1; -} - -module.exports = { - checkPathType, - getRelativePath, - getFolderTreeItem, - isPathInside, - getTestFolderPath, - calculateDepth -} diff --git a/src/utils/getArgs.js b/src/utils/getArgs.js index 8212e589..5caafee5 100644 --- a/src/utils/getArgs.js +++ b/src/utils/getArgs.js @@ -1,4 +1,4 @@ -const {PYTHAGORA_METADATA_DIR, CONFIG_FILENAME} = require("../const/common"); +const {PYTHAGORA_METADATA_DIR, CONFIG_FILENAME} = require("@pythagora.io/js-code-processing").common; const tryrequire = require("tryrequire"); const path = require('path'); const argsStr = process.env.PYTHAGORA_CONFIG || '';