diff --git a/rush.json b/rush.json index 6ef0234ff..c5099e4a5 100644 --- a/rush.json +++ b/rush.json @@ -2,7 +2,7 @@ "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush.schema.json", "npmVersion": "9.5.1", "rushVersion": "5.93.2", - "projectFolderMaxDepth": 4, + "projectFolderMaxDepth": 8, "projects": [ { "packageName": "@opentelemetry/sandbox-api", diff --git a/sandbox-tools/merge-repos/src/config.ts b/sandbox-tools/merge-repos/src/config.ts index 44a09bd93..09caa70d5 100644 --- a/sandbox-tools/merge-repos/src/config.ts +++ b/sandbox-tools/merge-repos/src/config.ts @@ -120,9 +120,24 @@ export const foldersToMerge: IMergePackageDetail[] = [ { name: "@opentelemetry/sdk-metrics", destPath: "pkgs/sdk/metrics/", srcPath: "auto-merge/js/packages/sdk-metrics/", bundleName: "otel-sndbx.sdk-metrics", bundleNamespace: "opentelemetry.sandbox.web.sdk-metrics" }, { name: "@opentelemetry/sdk-logs", destPath: "pkgs/sdk/logs/", srcPath: "auto-merge/js/experimental/packages/sdk-logs/", bundleName: "otel-sndbx.sdk-logs", bundleNamespace: "opentelemetry.sandbox.sdk-logs" }, { name: "@opentelemetry/otlp-exporter-base", destPath: "pkgs/exporters/otlp/base/", srcPath: "auto-merge/js/experimental/packages/otlp-exporter-base", bundleName: "otel-exprtr.otlp-base", bundleNamespace: "opentelemetry.sandbox.exporter.otlp-base" }, + { name: "@opentelemetry/otlp-transformer", destPath: "pkgs/exporters/otlp/transformer", srcPath: "auto-merge/js/experimental/packages/otlp-transformer", bundleName: "otel-exprtr.otlp-transformer", bundleNamespace: "opentelemetry.sandbox.exporter.otlp-transformer" }, + { name: "@opentelemetry/otlp-proto-exporter-base", destPath: "pkgs/exporters/otlp/proto-base", srcPath: "auto-merge/js/experimental/packages/otlp-proto-exporter-base", bundleName: "otel-exprtr.otlp-proto-base", bundleNamespace: "opentelemetry.sandbox.exporter.otlp-proto-base", noTests: true, + submodules: [ + { + path: "protos", + url: "https://github.com/open-telemetry/opentelemetry-proto.git" + } + ] + }, + // // { name: "@opentelemetry/exporter-trace-otlp-grpc", destPath: "pkgs/exporters/otlp/trace-grpc/", srcPath: "auto-merge/js/experimental/packages/exporter-trace-otlp-grpc/" }, // // { name: "@opentelemetry/exporter-jaeger", destPath: "pkgs/exporters/jaeger/", srcPath: "auto-merge/js/packages/opentelemetry-exporter-jaeger/" }, - // // { name: "@opentelemetry/exporter-zipkin", destPath: "pkgs/exporters/zipkin/", srcPath: "auto-merge/js/packages/opentelemetry-exporter-zipkin/" }, + { name: "@opentelemetry/exporter-zipkin", destPath: "pkgs/exporters/zipkin/", srcPath: "auto-merge/js/packages/opentelemetry-exporter-zipkin/", bundleName: "otel-exprtr.zipkin", bundleNamespace: "opentelemetry.sandbox.exporter.zipkin" }, + { name: "@opentelemetry/exporter-trace-otlp-proto", destPath: "pkgs/exporters/otlp/trace/proto/", srcPath: "auto-merge/js/experimental/packages/exporter-trace-otlp-proto", bundleName: "otel-exprtr.trace-otlp-proto", bundleNamespace: "opentelemetry.sandbox.exporter.trace-otlp-proto", noNodeTests: true }, + { name: "@opentelemetry/exporter-trace-otlp-http", destPath: "pkgs/exporters/otlp/trace/http/", srcPath: "auto-merge/js/experimental/packages/exporter-trace-otlp-http", bundleName: "otel-exprtr.trace-otlp-http", bundleNamespace: "opentelemetry.sandbox.exporter.trace-otlp-http" }, + { name: "@opentelemetry/exporter-metrics-otlp-http", destPath: "pkgs/exporters/otlp/metrics/http/", srcPath: "auto-merge/js/experimental/packages/opentelemetry-exporter-metrics-otlp-http", bundleName: "otel-exprtr.metrics-otlp-http", bundleNamespace: "opentelemetry.sandbox.exporter.metrics-otlp-http" }, + { name: "@opentelemetry/exporter-logs-otlp-proto", destPath: "pkgs/exporters/otlp/logs/proto/", srcPath: "auto-merge/js/experimental/packages/exporter-logs-otlp-proto", bundleName: "otel-exprtr.logs-otlp-proto", bundleNamespace: "opentelemetry.sandbox.exporter.logs-otlp-proto" }, + { name: "@opentelemetry/exporter-logs-otlp-http", destPath: "pkgs/exporters/otlp/logs/http/", srcPath: "auto-merge/js/experimental/packages/exporter-logs-otlp-http", bundleName: "otel-exprtr.logs-otlp-http", bundleNamespace: "opentelemetry.sandbox.exporter.logs-otlp-http" }, { name: "@opentelemetry/instrumentation", destPath: "pkgs/instrumentations/instrumentation/", srcPath: "auto-merge/js/experimental/packages/opentelemetry-instrumentation/", bundleName: "otel-sndbx.instrumentation", bundleNamespace: "opentelemetry.sandbox.web.instrumentation" }, { name: "@opentelemetry/instrumentation-fetch", destPath: "pkgs/instrumentations/web/fetch", srcPath: "auto-merge/js/experimental/packages/opentelemetry-instrumentation-fetch/", bundleName: "otel-sndbx.instr.fetch", bundleNamespace: "opentelemetry.sandbox.instr.web.fetch", noNodeTests: true }, { name: "@opentelemetry/instrumentation-xml-http-request", destPath: "pkgs/instrumentations/web/xhr", srcPath: "auto-merge/js/experimental/packages/opentelemetry-instrumentation-xml-http-request/", bundleName: "otel-sndbx.instr.xhr", bundleNamespace: "opentelemetry.sandbox.instr.web.xhr", noWorkerTests: true, noNodeTests: true }, @@ -132,7 +147,13 @@ export const foldersToMerge: IMergePackageDetail[] = [ { name: "@opentelemetry/auto-instrumentations-web", destPath: "pkgs/instrumentations/web/auto/", srcPath: "auto-merge/contrib/metapackages/auto-instrumentations-web/", bundleName: "otel-sndbx.instr-auto", bundleNamespace: "opentelemetry.sandbox.instr.web.auto", noWorkerTests: true, noNodeTests: true }, { name: "@opentelemetry/opentelemetry-browser-detector", destPath: "pkgs/detectors/browser/", srcPath: "auto-merge/js/experimental/packages/opentelemetry-browser-detector/", bundleName: "otel-sndbx.detect-browser", bundleNamespace: "opentelemetry.sandbox.detector.browser", noWorkerTests: true }, { name: "@opentelemetry/api-logs", destPath: "pkgs/api-logs/", srcPath: "auto-merge/js/experimental/packages/api-logs/", bundleName: "otel-sndbx.api-logs", bundleNamespace: "opentelemetry.sandbox.api-logs", noWorkerTests: true, noBrowserTests: true }, - { name: "@opentelemetry/api-events", destPath: "pkgs/api-events/", srcPath: "auto-merge/js/experimental/packages/api-events/", bundleName: "otel-sndbx.api-events", bundleNamespace: "opentelemetry.sandbox.api-events", noWorkerTests: true, noBrowserTests: true } + { name: "@opentelemetry/api-events", destPath: "pkgs/api-events/", srcPath: "auto-merge/js/experimental/packages/api-events/", bundleName: "otel-sndbx.api-events", bundleNamespace: "opentelemetry.sandbox.api-events", noWorkerTests: true, noBrowserTests: true }, + + // Examples + // Disabling the conversion of the contrib-js examples as there are conflicting version issues with webpack-dev-server that prevents the contrib version from running + //{ name: "web-examples", destPath: "examples/web/contrib/", srcPath: "auto-merge/contrib/examples/web/", bundleName: "otel-sndbx.contrib-examples", bundleNamespace: "opentelemetry.sandbox.contrib.examples", noTests: true, noLint: true, noVersion: true, noBuild: true }, + { name: "web-opentelemetry-example", destPath: "examples/web/js/", srcPath: "auto-merge/js/examples/opentelemetry-web/", bundleName: "otel-sndbx.js-examples", bundleNamespace: "opentelemetry.sandbox.js.examples", noTests: true, noLint: true, noVersion: true, noBuild: true } + ]; /** @@ -167,7 +188,8 @@ export let dependencyVersions = { "@types/jquery": "^3.5.14", "@typescript-eslint/eslint-plugin": "5.3.1", "@typescript-eslint/parser": "5.3.1", - "zone.js": "^0.11.4" + "zone.js": "^0.11.4", + "typescript": "^4.9.5" }; /** @@ -236,7 +258,8 @@ export let initDevDependencyVersions = { "@rollup/plugin-replace": "^5.0.2", "rollup-plugin-cleanup": "^3.2.1", "rollup-plugin-minify-es": "^1.1.1", - "uglify-js": "^3.17.4" + "uglify-js": "^3.17.4", + "typescript": "^4.9.5" }; /** diff --git a/sandbox-tools/merge-repos/src/github/github.ts b/sandbox-tools/merge-repos/src/github/github.ts index 08f47c96f..d79a6b83a 100644 --- a/sandbox-tools/merge-repos/src/github/github.ts +++ b/sandbox-tools/merge-repos/src/github/github.ts @@ -101,7 +101,7 @@ export async function gitHubCreateForkRepo(gitRoot: string, repoToFork: string) } } -export async function createPullRequest(git: SimpleGit, gitRoot: string, title: string, body: string, targetRepo: string, targetBranch) { +export async function createPullRequest(git: SimpleGit, gitRoot: string, title: string, body: string, targetRepo: string, targetBranch, test: boolean) { let status = await git.status(); let branchName = status.current; let tempBodyFile: string; @@ -120,6 +120,12 @@ export async function createPullRequest(git: SimpleGit, gitRoot: string, title: "--base", targetBranch ]; + if (test) { + prArgs.push("--draft"); + prArgs.push("--label"); + prArgs.push("do-not-merge"); + } + if (body) { if (body.length > 256) { tempBodyFile = process.cwd() + "/.prBody.txt"; diff --git a/sandbox-tools/merge-repos/src/mergeStagingToMain.ts b/sandbox-tools/merge-repos/src/mergeStagingToMain.ts index 293bd9df7..f3c49952a 100644 --- a/sandbox-tools/merge-repos/src/mergeStagingToMain.ts +++ b/sandbox-tools/merge-repos/src/mergeStagingToMain.ts @@ -117,6 +117,11 @@ let _packages: IPackages = { dest: {}, }; +/** + * An array of submodules paths created that should be ignored from validation + */ +let _subModules: string[] = []; + /** * Show the Help for this tool */ @@ -381,6 +386,16 @@ async function getStagingRepo(git: SimpleGit, repoName: string, details: IRepoDe await stagingGit.add(destPackage.pkgPath); } + if (packageDetails.submodules) { + for (let lp = 0; lp < packageDetails.submodules.length; lp++) { + let moduleDef = packageDetails.submodules[lp]; + let modulePath = path.join(packageDetails.destPath, moduleDef.path).replace(/\\/g, "/"); + log ("adding submodule " + modulePath + " => " + moduleDef.url); + await stagingGit.submoduleAdd(moduleDef.url, modulePath); + _subModules.push(modulePath); + } + } + log("Update package name references"); await updateFilePackageReferences(stagingDetails, forkDestOrg, packageDetails.destPath); }); @@ -427,6 +442,9 @@ async function updateRushJson(git: SimpleGit, thePath: string) { } rushJson.rushVersion = (_packages.dest[SANDBOX_PROJECT_NAME].pkg["devDependencies"]["@microsoft/rush"]).replace(/\^/, ""); + + // Set the rush max project depth + rushJson.projectFolderMaxDepth = 8; Object.keys(_packages.dest).forEach((packageKey) => { if (packageKey !== SANDBOX_PROJECT_NAME) { @@ -444,7 +462,7 @@ async function updateRushJson(git: SimpleGit, thePath: string) { rushJson.projects.push({ packageName: packageKey, projectFolder: _packages.dest[packageKey].rPath, - shouldPublish: true + shouldPublish: !_packages.dest[packageKey].pkg.private, }); } } @@ -532,7 +550,9 @@ function updateDependencies(srcPackage: IPackageJson, destPackage: IPackageJson, if (rootVersion && rootVersion !== srcVersion) { versionDiff = true; srcVersion = rootVersion; - } else if(dependencyVersions[key] && srcVersion !== dependencyVersions[key]) { + } + + if(dependencyVersions[key] && srcVersion !== dependencyVersions[key]) { // Always use these versions srcVersion = dependencyVersions[key]; versionDiff = true; @@ -624,6 +644,7 @@ function updatePackageJson(basePath: string, dest: string, srcPackage: IPackageJ function updatePackageJsonScripts(basePath: string, dest: string, newPackage: IPackageJson, srcPackage: IPackageJson, packageDetails: IMergePackageDetail) { let changed = false; let versionUpdate = path.relative(dest, path.join(basePath, "./scripts/version-update.js")).replace(/\\/g, "/"); + let protoGen = path.relative(dest, path.join(basePath, "./scripts/generate-protos.js")).replace(/\\/g, "/"); log(`Checking for ${path.join(basePath, path.join(packageDetails.destPath, "./tsconfig.all.json")).replace(/\\/g, "/")}`); let tsConfigJson = "tsconfig.all.json"; if (!fs.existsSync(path.join(basePath, path.join(packageDetails.destPath, "./tsconfig.all.json")).replace(/\\/g, "/"))) { @@ -660,6 +681,42 @@ function updatePackageJsonScripts(basePath: string, dest: string, newPackage: IP "lint:fix-quiet": "npm run lint -- --fix --quiet", "version": `node ${versionUpdate}`, "watch": "npm run version && tsc --build --watch " + tsConfigJson.trim() + "" + }; + + // Automatically add "npm run protos" to the prebuild if present + if (newPackage.scripts && newPackage.scripts["protos"]) { + if (!packageDetails.compileScripts || !packageDetails.compileScripts.pre) { + packageDetails.compileScripts = (packageDetails.compileScripts || {}); + packageDetails.compileScripts.pre = [ "protos" ]; + } + } + + if (packageDetails.compileScripts) { + if (packageDetails.compileScripts.pre) { + let preBuild = ""; + packageDetails.compileScripts.pre.forEach((value) => { + if (preBuild) { + preBuild += " && "; + } + preBuild += "npm run " + value; + }); + + newPackageScripts["pre-build"] = preBuild; + newPackageScripts["build"] = newPackageScripts["build"].replace("&& tsc --build ", "&& npm run pre-build && tsc --build "); + } + + if (packageDetails.compileScripts.post) { + let postBuild = ""; + packageDetails.compileScripts.post.forEach((value) => { + if (postBuild) { + postBuild += " && "; + } + postBuild += "npm run " + value; + }); + + newPackageScripts["post-build"] = postBuild; + newPackageScripts["build"] = newPackageScripts["build"].replace("&& npm run package", "&& npm run post-build && npm run package"); + } } if (!hasKarmaBrowserCfg && fs.existsSync(path.join(basePath, path.join(packageDetails.destPath, "./karma.conf.js")).replace(/\\/g, "/"))) { @@ -667,10 +724,15 @@ function updatePackageJsonScripts(basePath: string, dest: string, newPackage: IP hasKarmaBrowserCfg = true; } + if (!newPackage.scripts) { fail(null, JSON.stringify(newPackage)); } + if (newPackage.scripts["protos:generate"]) { + newPackage.scripts["protos:generate"] = `node ${protoGen}`; + } + Object.keys(newPackageScripts).forEach((script) => { if (!newPackage.scripts[script] || newPackage.scripts[script] !== newPackageScripts[script]) { let addTarget = true; @@ -688,6 +750,12 @@ function updatePackageJsonScripts(basePath: string, dest: string, newPackage: IP if (script.startsWith("test:node")) { addTarget = !packageDetails.noNodeTests; } + } else if (script.startsWith("build") || script.startsWith("compile") || script.startsWith("clean")) { + addTarget = !packageDetails.noBuild; + } else if (script.startsWith("lint")) { + addTarget = !packageDetails.noLint; + } else if (script.startsWith("version")) { + addTarget = !packageDetails.noVersion; } newPackage.scripts[script] = addTarget ? newPackageScripts[script] : ""; @@ -702,6 +770,13 @@ function updatePackageJsonScripts(basePath: string, dest: string, newPackage: IP } }); + if (packageDetails.scripts) { + Object.keys(packageDetails.scripts).forEach((script) => { + newPackage.scripts[script] = packageDetails.scripts[script]; + changed = true; + }); + } + return changed; } @@ -838,6 +913,19 @@ async function mergeStagingToMaster(mergeGit: SimpleGit, stagingDetails: IStagin return true; } + let destPath = path.join(destFolder, source).replace(/\\/g, "/"); + let isSubmodule = false; + _subModules.forEach((subPath) => { + if (destPath.indexOf(subPath) !== -1) { + log(` - Submodule ignore: ${subPath}`); + isSubmodule = true; + } + }); + + if (isSubmodule) { + return true; + } + return false; } @@ -1012,7 +1100,7 @@ localGit.checkIsRepo().then(async (isRepo) => { } if (prRequired && createPr && await pushToBranch(mergeGit)) { - await createPullRequest(mergeGit, _mergeGitRoot, prTitle, prBody, originRepo, destBranch); + await createPullRequest(mergeGit, _mergeGitRoot, prTitle, prBody, originRepo, destBranch, _theArgs.switches.test); try { // Attempt to push the tags to the origin diff --git a/sandbox-tools/merge-repos/src/support/types.ts b/sandbox-tools/merge-repos/src/support/types.ts index c866d57d4..c0dc7faa4 100644 --- a/sandbox-tools/merge-repos/src/support/types.ts +++ b/sandbox-tools/merge-repos/src/support/types.ts @@ -92,6 +92,11 @@ export interface IMergeDetail { optional?: boolean; } +export interface ISubModuleDef { + path: string, + url: string +} + export interface IMergePackageDetail extends IMergeDetail { /** * Identifies the package name. @@ -128,6 +133,46 @@ export interface IMergePackageDetail extends IMergeDetail { * Identifies that this package doesn't have any node tests and therefore should not add node test targets or config */ noNodeTests?: boolean; + + /** + * Identifies that this package doesn't have any build steps + */ + noBuild?: boolean; + + /** + * Identifies that this package doesn't support any linting and therefore should not add eslint targets or config + */ + noLint?: boolean; + + /** + * Identifies that this package doesn't support the version.ts and therefore should not add the automatic version targets or config + */ + noVersion?: boolean; + + /** + * Additional build targets that should be added to the package.json build step + */ + compileScripts?: { + /** + * The script names to add before the tsc compile + */ + pre?: string[], + + /** + * The script names to add after the tsc compile but before packaging + */ + post?: string[] + }, + + /** + * Optional additional scripts and overrides, applied after the standard conversions + */ + scripts?: { [key: string]: string }, + + /** + * Optionally defines any submodules that this package requires, each sub-module will be located within a sub-folder of the project + */ + submodules?: ISubModuleDef[], } /** @@ -135,6 +180,7 @@ export interface IMergePackageDetail extends IMergeDetail { */ export interface IPackageJson { name: string, + private?: boolean, version: string, description: string, keywords?: string[], diff --git a/sandbox-tools/merge-repos/src/syncReposToStaging.ts b/sandbox-tools/merge-repos/src/syncReposToStaging.ts index 87ceba7e1..1ce55acc9 100644 --- a/sandbox-tools/merge-repos/src/syncReposToStaging.ts +++ b/sandbox-tools/merge-repos/src/syncReposToStaging.ts @@ -526,7 +526,7 @@ localGit.checkIsRepo().then(async (isRepo) => { } if (prRequired && createPr && await pushToBranch(mergeGit)) { - await createPullRequest(mergeGit, _mergeGitRoot, prTitle, prBody, originRepo, originBranch); + await createPullRequest(mergeGit, _mergeGitRoot, prTitle, prBody, originRepo, originBranch, _theArgs.switches.test); try { // Attempt to push the tags to the origin