diff --git a/.github/workflows/test-lod-conversion.yml b/.github/workflows/test-lod-conversion.yml new file mode 100644 index 00000000..96c7f2c4 --- /dev/null +++ b/.github/workflows/test-lod-conversion.yml @@ -0,0 +1,83 @@ +name: Test LOD Conversion + +on: + pull_request: + +jobs: + build-and-run: + runs-on: windows-2019 + + steps: + - uses: actions/checkout@v2 + with: + lfs: true + + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '3.1.x' + + - name: Publish with dotnet + run: dotnet publish -c Release -r win10-x64 -o ./publish --self-contained true + + - name: Copy manifest builder to publish directory + run: | + Copy-Item -Path "${{ github.workspace }}\DCL_PiXYZ\bin\Release\netcoreapp3.1\win10-x64\scene-lod-entities-manifest-builder" -Destination "${{ github.workspace }}\publish\scene-lod-entities-manifest-builder" -Recurse + shell: pwsh + + - name: Copy road coordinates + run: | + Copy-Item -Path "${{ github.workspace }}\DCL_PiXYZ\bin\Release\netcoreapp3.1\win10-x64\RoadCoordinates.json" -Destination "${{ github.workspace }}\publish" + shell: pwsh + + - name: Install Vulkan SDK + uses: jakoch/install-vulkan-sdk-action@v1.0.0 + with: + vulkan_version: 1.3.268.0 + optional_components: com.lunarg.vulkan.vma + install_runtime: true + cache: true + destination: ${{ github.workspace }}/vulkan-sdt + + - name: Move Vulkan DLL to output directory + run: | + mv ./vulkan-sdt/1.3.268.0/runtime/x64/vulkan-1.dll ./publish/ + shell: bash + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '18.14.2' + + - name: Run the application + env: + PIXYZPRODUCTNAME: ${{ secrets.PIXYZPRODUCTNAME }} + PIXYZTOKEN: ${{ secrets.PIXYZTOKEN }} + run: .\publish\DCL_PiXYZ.exe "coords" "5,19" "${{ github.workspace }}\publish\OutputDirectoryPath" "${{ github.workspace }}\publish\scene-lod-entities-manifest-builder" "false" "true" + + - name: Check Output Files with Size Tolerance in KB + run: | + $basePath = "${{ github.workspace }}\publish\OutputDirectoryPath\5,19" + $files = @( + "bafkreictrpcnce5eoink3tdtrm74vgbniho7afl6xoi46lk3iag2u7aju4_0.fbx", # Hash output for 5,19 + "bafkreictrpcnce5eoink3tdtrm74vgbniho7afl6xoi46lk3iag2u7aju4_1.fbx", + "bafkreictrpcnce5eoink3tdtrm74vgbniho7afl6xoi46lk3iag2u7aju4_2.fbx" + ) + $expectedSizesB = @(6210224, 631424, 219776 ) # Expected sizes in bytes + $toleranceB = 3000 # 2 KB tolerance + + foreach ($i in 0..2) { + $filePath = Join-Path -Path $basePath -ChildPath $files[$i] + if (-Not (Test-Path $filePath)) { + throw "File not found: $filePath" + } + + $fileSizeInBytes = (Get-Item $filePath).length + $lowerBoundB = $expectedSizesB[$i] - $toleranceB + $upperBoundB = $expectedSizesB[$i] + $toleranceB + if ($fileSizeInBytes -lt $lowerBoundB -or $fileSizeInBytes -gt $upperBoundB) { + throw "File size mismatch for $filePath. Expected: $($expectedSizesB[$i]) B (+/- 3000 B), Found: $fileSizeInBytes bytes" + } + } + Write-Host "All files exist and are within the acceptable size range in KB." + shell: pwsh diff --git a/DCL_PiXYZ/PXZEntryPoint.cs b/DCL_PiXYZ/PXZEntryPoint.cs index 390ebae1..98ada1a5 100644 --- a/DCL_PiXYZ/PXZEntryPoint.cs +++ b/DCL_PiXYZ/PXZEntryPoint.cs @@ -9,6 +9,7 @@ using Newtonsoft.Json; using SceneImporter; using UnityEngine.Pixyz.API; +using UnityEngine.Pixyz.Core; namespace DCL_PiXYZ { @@ -27,57 +28,58 @@ private static async Task RunLODBuilder(string[] args) string defaultSceneLodManifestDirectory = Path.Combine(Directory.GetCurrentDirectory(), "scene-lod-entities-manifest-builder/"); bool isDebug = true; + bool installNPM = true; + if (args.Length > 0) { defaultScene = args[1]; defaultOutputPath = args[2]; defaultSceneLodManifestDirectory = args[3]; - isDebug = false; + bool.TryParse(args[4], out isDebug); + bool.TryParse(args[5], out installNPM); } //Conversion type can be single or bulk //If its single, we pass as many scenes as we want to parse separated by ; - //If its bulk, a single number will represent a square to parse, going from -value to value + //If its bulk, a single number will represent a square to parse, going from -value to value. + //Comment (Juani): Im living bulk implementation for reference, but currently Im invoking the creation of all the scenes through an external program + //PiXYZ was crashing and exiting the application if it was called form the same program //Scenes param is single coordinates or bulk value. Single scenes are separated by var sceneConversionInfo = new SceneConversionInfo("7000;3000;1000", "triangle", "coords", "single", defaultScene); var pathHandler = new SceneConversionPathHandler(isDebug, defaultOutputPath, defaultSceneLodManifestDirectory, "SuccessScenes.txt", "FailScenes.txt", "PolygonCount.txt" , "FailedGLBImport.txt" , defaultScene); List roadCoordinates = LoadRoads(); + var convertedScenes = LoadConvertedScenes(); CreateDirectories(sceneConversionInfo); - FrameworkInitialization(pathHandler.ManifestProjectDirectory); + FrameworkInitialization(pathHandler.ManifestProjectDirectory, installNPM); foreach (string currentScene in sceneConversionInfo.ScenesToAnalyze) { if (IsRoad(roadCoordinates, currentScene)) continue; - - if (HasSceneBeenAnalyzed(sceneConversionInfo.AnalyzedScenes, currentScene)) continue; + + if (HasSceneBeenAnalyzed(convertedScenes, currentScene)) continue; sceneConversionInfo.SceneImporter = new SceneImporter(sceneConversionInfo.ConversionType, currentScene, sceneConversionInfo.WebRequestsHandler); if (!await SceneDefinitionDownloadSuccesfully(sceneConversionInfo, currentScene, pathHandler)) continue; - pathHandler.SetOutputPath(sceneConversionInfo.SceneImporter); - - if (HasSceneBeenConverted(pathHandler, currentScene)) continue; + if (HasSceneBeenConverted(convertedScenes, sceneConversionInfo.SceneImporter.GetSceneBasePointer())) continue; if (CheckEmptyScene(sceneConversionInfo.SceneImporter.GetCurrentScenePointersList(), currentScene)) continue; + pathHandler.SetOutputPath(sceneConversionInfo.SceneImporter); + //Add it to the analyzed scenes array foreach (string pointer in sceneConversionInfo.SceneImporter.GetCurrentScenePointersList()) - sceneConversionInfo.AnalyzedScenes.Add(pointer); - - if (CheckFaillingDebugScenes(sceneConversionInfo.SceneImporter.GetCurrentScenePointersList(), currentScene)) continue; + convertedScenes.Add(pointer); + Console.WriteLine("BEGIN SCENE CONVERSION FOR " + currentScene); if (!await ManifestGeneratedSuccesfully(sceneConversionInfo, pathHandler, currentScene)) continue; - if (!await sceneConversionInfo.SceneImporter.DownloadAllContent(pathHandler)) continue; - - Console.WriteLine("Begin scene conversion for " + currentScene); var pxzParams = new PXZParams { - DecimationType = sceneConversionInfo.DecimationType, ParcelAmount = sceneConversionInfo.SceneImporter.GetCurrentScenePointersList().Length, SceneContent = sceneConversionInfo.SceneImporter.sceneContent, SceneHash = sceneConversionInfo.SceneImporter.GetSceneHash(), - ScenePointer = sceneConversionInfo.SceneImporter.GetSceneBasePointer() + DecimationType = sceneConversionInfo.DecimationType, ParcelAmount = sceneConversionInfo.SceneImporter.GetCurrentScenePointersList().Length, SceneContent = sceneConversionInfo.SceneImporter.sceneContent }; foreach (var decimationValue in sceneConversionInfo.DecimationToAnalyze) { @@ -87,8 +89,21 @@ private static async Task RunLODBuilder(string[] args) pxzParams.LodLevel += 1; } GC.Collect(); - Console.WriteLine("END SCENE CONVERSION FOR SCENE " + currentScene); + Console.WriteLine("END SCENE CONVERSION FOR " + currentScene); + UpdateConvertedScenesFile(convertedScenes); } + DoManifestCleanup(isDebug, pathHandler); + } + + private static void DoManifestCleanup(bool isDebug, SceneConversionPathHandler pathHandler) + { + if (isDebug) + return; + + var dir = new DirectoryInfo(pathHandler.ManifestOutputJsonDirectory); + + foreach (var fi in dir.GetFiles()) + fi.Delete(); } private static async Task DoConversion(PXZParams pxzParams, SceneConversionInfo sceneConversionInfo, string scene, SceneConversionPathHandler pathHandler) @@ -100,8 +115,7 @@ private static async Task DoConversion(PXZParams pxzParams, SceneConversionInfo //Check if they were converted stopwatch.Restart(); Console.WriteLine($"BEGIN CONVERTING {scene} WITH {pxzParams.DecimationValue}"); - await ConvertScene(pxzParams, pathHandler); - Console.WriteLine($"END CONVERTING {scene} WITH {pxzParams.DecimationValue}"); + await ConvertScene(pxzParams, pathHandler, sceneConversionInfo); stopwatch.Stop(); string elapsedTime = string.Format("{0:00}:{1:00}:{2:00}", @@ -115,10 +129,9 @@ private static async Task DoConversion(PXZParams pxzParams, SceneConversionInfo } } - private static bool HasSceneBeenConverted(SceneConversionPathHandler pathHandler, string scene) + private static bool HasSceneBeenConverted(List convertedScenes, string scene) { - var d = new DirectoryInfo(pathHandler.OutputPath); - if (d.Exists && d.GetFiles().Length > 0) + if (convertedScenes.Contains(scene)) { Console.WriteLine($"Skipping scene {scene} since its already converted"); return true; @@ -153,17 +166,6 @@ private static async Task SceneDefinitionDownloadSuccesfully(SceneConversi return true; } - private static bool CheckFaillingDebugScenes(string[] currentPointersList, string scene) - { - if (currentPointersList[0].Equals("-15,-39") || currentPointersList[0].Equals("-27,-17") || currentPointersList[0].Equals("-75,-9") || currentPointersList[0].Equals("-5,36") || currentPointersList[0].Equals("16,34")) - { - Console.WriteLine($"Skipping scene {scene} because it was causing an exit without exception"); - return true; - } - - return false; - } - private static bool CheckEmptyScene(string[] currentPointersList, string scene) { //Check empty scenes @@ -206,31 +208,28 @@ private static async Task GenerateManifest(string sceneType, string sceneV } } - Console.WriteLine($"END MANIFEST GENERATION FOR SCENE {sceneValue}"); return true; // Return true as default, indicating success if no unignorable error was found. } - private static async Task ConvertScene(PXZParams pxzParams, SceneConversionPathHandler pathHandler) + private static async Task ConvertScene(PXZParams pxzParams, SceneConversionPathHandler pathHandler, SceneConversionInfo sceneConversionInfo) { SceneRepositioner.SceneRepositioner sceneRepositioner = new SceneRepositioner.SceneRepositioner(pathHandler.ManifestOutputJsonFile, pxzParams.SceneContent, pxz, pathHandler, pxzParams.LodLevel); List models = await sceneRepositioner.SetupSceneInPiXYZ(); - List modifiers = new List(); modifiers.Add(new PXZBeginCleanMaterials()); modifiers.Add(new PXZRepairMesh(models)); - if (pxzParams.LodLevel != 0) { modifiers.Add(new PXZDeleteByName(".*collider.*")); - modifiers.Add(new PXZDecimator(pxzParams.ScenePointer, pxzParams.DecimationType, + modifiers.Add(new PXZDecimator(sceneConversionInfo.SceneImporter.GetSceneBasePointer(), pxzParams.DecimationType, pxzParams.DecimationValue, pxzParams.ParcelAmount, pathHandler)); modifiers.Add(new PXZMergeMeshes(pxzParams.LodLevel)); } - modifiers.Add(new PXZExporter(pxzParams, pathHandler)); + modifiers.Add(new PXZExporter(pxzParams, pathHandler, sceneConversionInfo)); PXZStopwatch stopwatch = new PXZStopwatch(); foreach (var pxzModifier in modifiers) @@ -257,31 +256,50 @@ private static List LoadRoads() string filePath = Path.Combine(Directory.GetCurrentDirectory(), "RoadCoordinates.json"); return JsonConvert.DeserializeObject>(File.ReadAllText(filePath)); } - - private static void FrameworkInitialization(string sceneManifestDirectory) + + private static List LoadConvertedScenes() + { + string convertedScenePathFile = Path.Combine(Directory.GetCurrentDirectory(), "ConvertedScenes.json"); + if (File.Exists(convertedScenePathFile)) + { + return JsonConvert.DeserializeObject>(File.ReadAllText(convertedScenePathFile)); + } + + return new List(); + } + + private static void UpdateConvertedScenesFile(List convertedScenes) + { + string convertedScenePathFile = Path.Combine(Directory.GetCurrentDirectory(), "ConvertedScenes.json"); + File.WriteAllText(convertedScenePathFile, JsonConvert.SerializeObject(convertedScenes)); + } + + private static void FrameworkInitialization(string sceneManifestDirectory, bool installAndBuildNPM) { - Console.WriteLine("INSTALLING AND BUILDING NPM"); - NPMUtils.DoNPMInstall(sceneManifestDirectory); - Console.WriteLine("END INSTALLING AND BUILDING NPM"); + if (installAndBuildNPM) + { + Console.WriteLine("INSTALLING AND BUILDING NPM"); + NPMUtils.DoNPMInstall(sceneManifestDirectory); + } Console.WriteLine("INITIALIZING PIXYZ"); InitializePiXYZ(); - Console.WriteLine("END INITIALIZING PIXYZ"); } private static void InitializePiXYZ() { - pxz = - PiXYZAPI.Initialize("PixyzSDKCSharp", - "204dda67aa3ea8bcb22a76bff9aa1224823b253144396405300e235e434c4711591892c19069c7"); + pxz = PiXYZAPI.Initialize(Environment.GetEnvironmentVariable("PIXYZPRODUCTNAME"), Environment.GetEnvironmentVariable("PIXYZTOKEN")); + + foreach (string s in pxz.Core.ListTokens().list) + pxz.Core.AddWantedToken(s); + // if no license is found, try to configure a license server if (!pxz.Core.CheckLicense()) - pxz.Core.InstallLicense("pixyzsdk-29022024.lic"); + pxz.Core.ConfigureLicenseServer("18.204.36.86", 27000); } private static void CreateDirectories(SceneConversionInfo sceneConversionInfo) { Directory.CreateDirectory(PXYZConstants.RESOURCES_DIRECTORY); - } public static void CloseApplication(string errorMessage) diff --git a/DCL_PiXYZ/PXZParams.cs b/DCL_PiXYZ/PXZParams.cs index ff4c4893..c0da29ed 100644 --- a/DCL_PiXYZ/PXZParams.cs +++ b/DCL_PiXYZ/PXZParams.cs @@ -8,8 +8,6 @@ namespace DCL_PiXYZ { public struct PXZParams { - public string SceneHash { get; set; } - public string ScenePointer { get; set; } public Dictionary SceneContent { get; set; } public int ParcelAmount { get; set; } public double DecimationValue { get; set; } @@ -25,7 +23,6 @@ public struct SceneConversionInfo public string DecimationType { get; } public string DecimationValues { get; } public List ScenesToAnalyze { get; set; } - public List AnalyzedScenes { get; set; } public List DecimationToAnalyze { get; set; } public SceneImporter SceneImporter; @@ -40,7 +37,6 @@ public SceneConversionInfo(string decimationValues, string decimationType, strin DecimationType = decimationType; DecimationValues = decimationValues; ScenesToAnalyze = new List(); - AnalyzedScenes = new List(); DecimationToAnalyze = new List(); SceneImporter = null; WebRequestsHandler = new WebRequestsHandler(); @@ -95,19 +91,17 @@ public struct SceneConversionPathHandler public string FailGLBImporterFile; public string OutputPath; public string ManifestOutputJsonFile; + public string ManifestOutputJsonDirectory; public string ManifestProjectDirectory; private readonly string DefaultOutputPath; - public bool IsDebug; - public SceneConversionPathHandler(bool isDebug, string defaultOutputPath, string manifestProjectDirectory, string successFile, string failFile, string vertexCountFile, string failGlbImporterFile, string scene) { - IsDebug = isDebug; DefaultOutputPath = defaultOutputPath; ManifestProjectDirectory = manifestProjectDirectory; - if (IsDebug) + if (isDebug) { SuccessFile = Path.Combine(defaultOutputPath, successFile); FailFile = Path.Combine(defaultOutputPath, failFile); @@ -125,12 +119,14 @@ public SceneConversionPathHandler(bool isDebug, string defaultOutputPath, string OutputPath = ""; ManifestOutputJsonFile = ""; + ManifestOutputJsonDirectory = ""; } public void SetOutputPath(SceneImporter sceneSceneImporter) { OutputPath = Path.Combine(DefaultOutputPath, sceneSceneImporter.GetSceneBasePointer()); - ManifestOutputJsonFile = Path.Combine(ManifestProjectDirectory, "output-manifests", sceneSceneImporter.GetSceneHash() + "-lod-manifest.json"); + ManifestOutputJsonDirectory = Path.Combine(ManifestProjectDirectory, "output-manifests"); + ManifestOutputJsonFile = Path.Combine(ManifestOutputJsonDirectory, sceneSceneImporter.GetSceneHash() + "-lod-manifest.json"); Directory.CreateDirectory(DefaultOutputPath); Directory.CreateDirectory(OutputPath); diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZBeginCleanMaterials.cs b/DCL_PiXYZ/PiXYZWorflow/PXZBeginCleanMaterials.cs index bd7c6f49..cb8a8b0a 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZBeginCleanMaterials.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZBeginCleanMaterials.cs @@ -56,7 +56,6 @@ uint copyMaterial } }*/ - Console.WriteLine("END PXZ CLEAN MATERIALS"); } } } \ No newline at end of file diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZDecimateAndBake.cs b/DCL_PiXYZ/PiXYZWorflow/PXZDecimateAndBake.cs index ce6b60c1..6a226f7e 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZDecimateAndBake.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZDecimateAndBake.cs @@ -91,7 +91,6 @@ public async Task ApplyModification(PiXYZAPI pxz) { pxz.Core.SetProperty(occurence, "Name", "DecimateTargetBake " + pxz.Core.GetProperty(occurence, "Id")); }*/ - Console.WriteLine("END PXZ DECIMATE AND BAKE"); } } } \ No newline at end of file diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZDecimator.cs b/DCL_PiXYZ/PiXYZWorflow/PXZDecimator.cs index f381ae7c..affd2305 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZDecimator.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZDecimator.cs @@ -42,7 +42,6 @@ public async Task ApplyModification(PiXYZAPI pxz) pxz.Scene.GetPolygonCount(new OccurrenceList(new uint[] { pxz.Scene.GetRoot() }), true); pxz.Algo.DecimateTarget(new OccurrenceList(new uint[]{pxz.Scene.GetRoot()}), decimate); WriteFinalVertexAmount(pxz.Scene.GetPolygonCount(new OccurrenceList(new uint[] { pxz.Scene.GetRoot() }),true)); - Console.WriteLine("END PXZ MODIFIER DECIMATOR"); } private void WriteFinalVertexAmount(ulong polygonCount) diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZDeleteByName.cs b/DCL_PiXYZ/PiXYZWorflow/PXZDeleteByName.cs index 1fa78ccf..6224983b 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZDeleteByName.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZDeleteByName.cs @@ -19,10 +19,9 @@ public async Task ApplyModification(PiXYZAPI pxz) { Console.WriteLine("BEGIN PXZ DELETE BY NAME FOR REGEX " + regex); - OccurrenceList occurenceToDelete = pxz.Scene.FindOccurrencesByProperty("Name", regex); + var occurenceToDelete = pxz.Scene.FindOccurrencesByProperty("Name", regex, caseInsensitive: true); foreach (uint u in occurenceToDelete.list) pxz.Scene.DeleteComponentByType(ComponentType.Part, u); - Console.WriteLine("END PXZ DELETE BY NAME FOR REGEX " + regex); } } } \ No newline at end of file diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZExporter.cs b/DCL_PiXYZ/PiXYZWorflow/PXZExporter.cs index 135dc640..cc9aa2ce 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZExporter.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZExporter.cs @@ -18,12 +18,12 @@ public class PXZExporter : IPXZModifier private readonly int lodLevel; private readonly SceneConversionPathHandler pathHandler; - public PXZExporter(PXZParams pxzParams, SceneConversionPathHandler pathHandler) + public PXZExporter(PXZParams pxzParams, SceneConversionPathHandler pathHandler, SceneConversionInfo sceneConversionInfo) { this.pathHandler = pathHandler; extensions = new List() { ".fbx", ".glb" }; path = pathHandler.OutputPath; - filename = $"{pxzParams.SceneHash}_{pxzParams.LodLevel}"; + filename = $"{sceneConversionInfo.SceneImporter.GetSceneHash()}_{pxzParams.LodLevel}"; lodLevel = pxzParams.LodLevel; } @@ -69,7 +69,6 @@ private void DoExportWithExtension(PiXYZAPI pxz, string extension) }), 1); } pxz.IO.ExportScene(Path.Combine(path, $"{filename}{extension}"), pxz.Scene.GetRoot()); - Console.WriteLine("END PXZ EXPORT "); } } } \ No newline at end of file diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZMergeMeshes.cs b/DCL_PiXYZ/PiXYZWorflow/PXZMergeMeshes.cs index 04db3476..1c944a31 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZMergeMeshes.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZMergeMeshes.cs @@ -53,7 +53,6 @@ public async Task ApplyModification(PiXYZAPI pxz) { Console.WriteLine(e); } - Console.WriteLine("END PXZ MERGE MESHES"); } private void MergeSubMeshes(OccurrenceList listToMerge, bool isOpaque) @@ -103,7 +102,6 @@ private void DoMerge(ulong currentVertexCount, OccurrenceList toMerge, bool isOp uint combineMeshes = pxz.Algo.CombineMeshes(toMerge, bakeOption); pxz.Core.SetProperty(combineMeshes, "Name", $"MERGED MESH {index} {(isOpaque ? "OPAQUE" : "FORCED_TRANSPARENT")}"); - Console.WriteLine("End merging meshes "); Console.WriteLine("Copying Material"); //Apply a copy of the material not to lose the reference diff --git a/DCL_PiXYZ/PiXYZWorflow/PXZRepairMesh.cs b/DCL_PiXYZ/PiXYZWorflow/PXZRepairMesh.cs index 66f99c9e..27546253 100644 --- a/DCL_PiXYZ/PiXYZWorflow/PXZRepairMesh.cs +++ b/DCL_PiXYZ/PiXYZWorflow/PXZRepairMesh.cs @@ -24,7 +24,6 @@ public async Task ApplyModification(PiXYZAPI pxz) if (pxzModel.needsRepair) pxz.Algo.RepairMesh(new OccurrenceList(new uint[]{pxzModel.modelOcurrence}), 0.1, true, false); } - Console.WriteLine($"END PXZ MESH REPAIR"); } } } \ No newline at end of file diff --git a/DCL_PiXYZ/SceneImporter/SceneImporter.cs b/DCL_PiXYZ/SceneImporter/SceneImporter.cs index 94f5a27a..aeb8ed36 100644 --- a/DCL_PiXYZ/SceneImporter/SceneImporter.cs +++ b/DCL_PiXYZ/SceneImporter/SceneImporter.cs @@ -62,7 +62,6 @@ public async Task DownloadSceneDefinition() SetResult(sceneDefinitions[0].id); } } - Console.WriteLine("END SCENE DEFINITION DOWNLOAD"); } catch (Exception e) { @@ -103,7 +102,6 @@ public async Task DownloadAllContent(SceneConversionPathHandler pathHandle return false; } } - Console.WriteLine("END FILE CONTENT DOWNLOAD"); return true; } diff --git a/DCL_PiXYZ/SceneRepositioner/JsonParsing/MeshRendererData.cs b/DCL_PiXYZ/SceneRepositioner/JsonParsing/MeshRendererData.cs index 2ffd97db..6fa2416b 100644 --- a/DCL_PiXYZ/SceneRepositioner/JsonParsing/MeshRendererData.cs +++ b/DCL_PiXYZ/SceneRepositioner/JsonParsing/MeshRendererData.cs @@ -19,7 +19,7 @@ public class MeshRendererData : ComponentData [Serializable] public abstract class DCLMesh { - public abstract PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary contentTable, SceneConversionPathHandler pathHandler, int lodLevel); + public abstract PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary contentTable, SceneConversionPathHandler pathHandler); } public struct PXZModel diff --git a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLGLTFMesh.cs b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLGLTFMesh.cs index 6a5bdbd9..2bddffe7 100644 --- a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLGLTFMesh.cs +++ b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLGLTFMesh.cs @@ -18,114 +18,128 @@ namespace AssetBundleConverter.LODs { public class DCLGLTFMesh : DCLMesh { - private SceneConversionPathHandler _pathHandler; + private SceneConversionPathHandler pathHandler; - private string src; + private readonly string src; public DCLGLTFMesh(string src) { this.src = src; } - - public override PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary contentTable, SceneConversionPathHandler pathHandler, int lodLevel) + + public override PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary contentTable, SceneConversionPathHandler pathHandler) { + this.pathHandler = pathHandler; + if (!contentTable.TryGetValue(src.ToLower(), out string modelPath)) { - Console.WriteLine($"ERROR: GLTF {src} file not found in sceneContent"); + LogError($"ERROR: GLTF {src} file not found in sceneContent"); return PXYZConstants.EMPTY_MODEL; } - bool modelWorkSuccessfull = true; + bool modelRecreatedSuccessfully = true; + try { - var readSettings = new ReadSettings(ValidationMode.TryFix); - var model = ModelRoot.Load(modelPath, readSettings); - foreach (var gltfMaterial in model.LogicalMaterials) - { - if (gltfMaterial.Alpha != AlphaMode.OPAQUE && !gltfMaterial.Name.Contains("FORCED_TRANSPARENT")) - gltfMaterial.Name += "FORCED_TRANSPARENT"; - } + ModifyModelMaterials(modelPath); + ExportModifiedModel(modelPath); + } + catch (Exception e) + { + LogError($"ERROR pre-processing GLTF with GLTFSharp for file {src}: {e}"); + modelRecreatedSuccessfully = false; + } - if (Path.GetExtension(modelPath).Contains("glb", StringComparison.OrdinalIgnoreCase)) - model.SaveGLB(modelPath); - else - model.SaveGLTF(modelPath); + try + { + uint importedFileOccurrence = pxz.IO.ImportScene(modelRecreatedSuccessfully ? modelPath + "_EDITED.glb" : modelPath); + pxz.Scene.SetParent(importedFileOccurrence, parent); + return new PXZModel(true, importedFileOccurrence); } catch (Exception e) { - //PXZEntryPoint.WriteToFile($"MODEL FAILED A {modelPath} {e}", "FAILEDIMPORTMODELS.txt"); - Console.WriteLine($"ERROR pre-processing GLTF material: {e}"); + Console.WriteLine($"ERROR: Importing GLTF {src} failed with error {e}"); + return PXYZConstants.EMPTY_MODEL; } - try - { - ReadSettings readSettings = new ReadSettings(ValidationMode.TryFix); - var model = ModelRoot.Load(modelPath, readSettings); - ModelRoot modelRoot = ModelRoot.CreateModel(); - Scene sceneModel = modelRoot.UseScene("Default"); - foreach (var modelLogicalNode in model.LogicalNodes) - { - if (modelLogicalNode.Mesh != null) - { - foreach(MeshPrimitive meshPrimitive in modelLogicalNode.Mesh.Primitives) - { - Mesh meshToExport = modelRoot.CreateMesh(modelLogicalNode.Name); - var positions = meshPrimitive.GetVertexAccessor("POSITION").AsVector3Array().ToArray(); - var normals = meshPrimitive.GetVertexAccessor("NORMAL").AsVector3Array().ToArray(); - var indices = (int[])(object)meshPrimitive.GetIndexAccessor().AsIndicesArray().ToArray(); - if (meshPrimitive.GetVertexAccessor("TEXCOORD_0") != null) - { - var texcoord = meshPrimitive.GetVertexAccessor("TEXCOORD_0").AsVector2Array().ToArray(); - meshToExport.CreatePrimitive() - .WithVertexAccessor("POSITION", positions) - .WithVertexAccessor("NORMAL", normals) - .WithVertexAccessor("TEXCOORD_0", texcoord) - .WithIndicesAccessor(PrimitiveType.TRIANGLES, indices) - .WithMaterial(modelRoot.CreateMaterial(meshPrimitive.Material.ToMaterialBuilder())); - } - else - { - meshToExport.CreatePrimitive() - .WithVertexAccessor("POSITION", positions) - .WithVertexAccessor("NORMAL", normals) - .WithIndicesAccessor(PrimitiveType.TRIANGLES, indices) - .WithMaterial(modelRoot.CreateMaterial(meshPrimitive.Material.ToMaterialBuilder())); - } - Node node = sceneModel.CreateNode(modelLogicalNode.Name).WithMesh(meshToExport); - node.WorldMatrix = modelLogicalNode.WorldMatrix; - } - } - } - modelRoot.SaveGLB(modelPath + "_EDITED.glb"); - } - catch (Exception e) - { - //PXZEntryPoint.WriteToFile($"MODEL FAILED B {modelPath} {e}", "FAILEDIMPORTMODELS.txt"); - modelWorkSuccessfull = false; - Console.WriteLine($"ERROR pre-processing GLTF: {e}"); - } - try + } + + private void ModifyModelMaterials(string modelPath) + { + var readSettings = new ReadSettings(ValidationMode.TryFix); + var model = ModelRoot.Load(modelPath, readSettings); + foreach (var gltfMaterial in model.LogicalMaterials) + { + if (gltfMaterial.Alpha != AlphaMode.OPAQUE && !gltfMaterial.Name.Contains("FORCED_TRANSPARENT")) + gltfMaterial.Name += "FORCED_TRANSPARENT"; + } + + SaveModel(model, modelPath); + } + + private void SaveModel(ModelRoot model, string modelPath) + { + if (Path.GetExtension(modelPath).Contains("glb", StringComparison.OrdinalIgnoreCase)) + model.SaveGLB(modelPath); + else + model.SaveGLTF(modelPath); + } + + private void ExportModifiedModel(string modelPath) + { + // Determine the output file path based on the original model path + string outputFile = $"{modelPath}_EDITED.glb"; + + // Check if the file already exists + if (File.Exists(outputFile)) + { + Console.WriteLine($"The file {outputFile} already exists. Skipping export."); + return; + } + + var readSettings = new ReadSettings(ValidationMode.TryFix); + var model = ModelRoot.Load(modelPath, readSettings); + var modelRoot = ModelRoot.CreateModel(); + var sceneModel = modelRoot.UseScene("Default"); + foreach (var modelLogicalNode in model.LogicalNodes) + { + if (modelLogicalNode.Mesh != null) { - if (lodLevel != 0) - { - uint importedFileOccurrence = pxz.IO.ImportScene(modelPath); - pxz.Scene.SetParent(importedFileOccurrence, parent); - return new PXZModel(true, importedFileOccurrence); - } - else + foreach (var meshPrimitive in modelLogicalNode.Mesh.Primitives) { - uint importedFileOccurrence = pxz.IO.ImportScene(modelWorkSuccessfull ? modelPath + "_EDITED.glb" : modelPath); - pxz.Scene.SetParent(importedFileOccurrence, parent); - return new PXZModel(true, importedFileOccurrence); + var meshToExport = modelRoot.CreateMesh(modelLogicalNode.Name); + BuildMesh(meshToExport.CreatePrimitive(), meshPrimitive.GetVertexAccessor("POSITION"), + meshPrimitive.GetVertexAccessor("NORMAL"), + meshPrimitive.GetVertexAccessor("TEXCOORD_0"), + (int[])(object)meshPrimitive.GetIndexAccessor().AsIndicesArray().ToArray(), + modelRoot.CreateMaterial(meshPrimitive.Material.ToMaterialBuilder())); + var node = sceneModel.CreateNode(modelLogicalNode.Name).WithMesh(meshToExport); + node.WorldMatrix = modelLogicalNode.WorldMatrix; } } - catch(Exception e) - { - Console.WriteLine($"ERROR: Importing GLTF {src} failed with error {e}"); - return PXYZConstants.EMPTY_MODEL; - } + } + modelRoot.SaveGLB(modelPath + "_EDITED.glb"); } + + private void BuildMesh(MeshPrimitive meshToExport, Accessor positions, Accessor normals, Accessor texcoord, int[] indices, Material material) + { + if (positions != null) + meshToExport.WithVertexAccessor("POSITION", positions.AsVector3Array().ToArray()); + + if (normals != null) + meshToExport.WithVertexAccessor("NORMAL", normals.AsVector3Array().ToArray()); + if (texcoord != null) + meshToExport.WithVertexAccessor("TEXCOORD_0", texcoord.AsVector2Array().ToArray()); + + meshToExport.WithIndicesAccessor(PrimitiveType.TRIANGLES, indices); + meshToExport.WithMaterial(material); + } + + private void LogError(string message) + { + FileWriter.WriteToFile(message, pathHandler.FailGLBImporterFile); + } } diff --git a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLPrimitiveMesh.cs b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLPrimitiveMesh.cs index e6041d96..9db4bdaf 100644 --- a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLPrimitiveMesh.cs +++ b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLPrimitiveMesh.cs @@ -14,7 +14,7 @@ public abstract class DCLPrimitiveMesh : DCLMesh { protected abstract uint GetMesh(PiXYZAPI pxz, string entityID); - public override PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary sceneContent, SceneConversionPathHandler pathHandler, int lodLevel) + public override PXZModel InstantiateMesh(PiXYZAPI pxz, string entityID, uint parent, uint material, Dictionary sceneContent, SceneConversionPathHandler pathHandler) { uint mesh = GetMesh(pxz, entityID); Matrix4 matrix4 = new Matrix4(); diff --git a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLRendereableEntity.cs b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLRendereableEntity.cs index d25c933b..776db03b 100644 --- a/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLRendereableEntity.cs +++ b/DCL_PiXYZ/SceneRepositioner/SceneBuilder/Entities/DCLRendereableEntity.cs @@ -49,13 +49,13 @@ public void InitEntity(PiXYZAPI pxz, uint pxzRootOcurrence) this.pxz = pxz; } - public PXZModel PositionAndInstantiteMesh(Dictionary contentTable, Dictionary renderableEntities, SceneConversionPathHandler pathHandler, int lodLevel) + public PXZModel PositionAndInstantiteMesh(Dictionary contentTable, Dictionary renderableEntities, SceneConversionPathHandler pathHandler) { InstantiateTransform(renderableEntities); if (rendereableMesh != null) { uint material = dclMaterial.GetMaterial(pxz, entityID.ToString(), contentTable); - return rendereableMesh.InstantiateMesh(pxz, entityID.ToString(), instantiatedEntity, material, contentTable, pathHandler, lodLevel); + return rendereableMesh.InstantiateMesh(pxz, entityID.ToString(), instantiatedEntity, material, contentTable, pathHandler); } else return PXYZConstants.EMPTY_MODEL; diff --git a/DCL_PiXYZ/SceneRepositioner/SceneRepositioner.cs b/DCL_PiXYZ/SceneRepositioner/SceneRepositioner.cs index 63b7e2b2..e8dce9d8 100644 --- a/DCL_PiXYZ/SceneRepositioner/SceneRepositioner.cs +++ b/DCL_PiXYZ/SceneRepositioner/SceneRepositioner.cs @@ -49,9 +49,8 @@ public async Task> SetupSceneInPiXYZ() dclRendereableEntity.Value.InitEntity(pxz, rootOccurrence); foreach (KeyValuePair dclRendereableEntity in renderableEntitiesDictionary) - models.Add(dclRendereableEntity.Value.PositionAndInstantiteMesh(sceneContent, renderableEntitiesDictionary, _pathHandler, lodLevel)); + models.Add(dclRendereableEntity.Value.PositionAndInstantiteMesh(sceneContent, renderableEntitiesDictionary, _pathHandler)); - Console.WriteLine("END REPOSITIONING"); return models; } diff --git a/Dockerfile b/Dockerfile index 97612d43..725a777f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,6 @@ COPY --from=dotnet-build /build/publish/ . WORKDIR /app COPY RoadCoordinates.json ./ -COPY ./pixyzsdk-29022024.lic ./pixyzsdk-29022024.lic COPY --from=scene-lod-build /scene-lod/dist ./scene-lod/dist COPY --from=scene-lod-build /scene-lod/package.json ./scene-lod/package.json COPY --from=scene-lod-build /scene-lod/package-lock.json ./scene-lod/package-lock.json diff --git a/PS1_Files/BulkConversion.ps1 b/PS1_Files/BulkConversion.ps1 new file mode 100644 index 00000000..962b1d86 --- /dev/null +++ b/PS1_Files/BulkConversion.ps1 @@ -0,0 +1,31 @@ +# Get the directory of the current script +$scriptDirectory = Split-Path -Parent -Path $MyInvocation.MyCommand.Definition + +# Construct the path to the directory on the same level as the script +# Replace 'OutputDirectoryPath' with the actual directory name you're targeting +$outputDirectoryPath = Join-Path -Path $scriptDirectory -ChildPath "built-lods" + +# Construct the path to the directory on the same level as the script +$manifestDirectoryPath = Join-Path -Path $scriptDirectory -ChildPath "scene-lod-entities-manifest-builder" + +# We install the manifest project +Set-Location -Path $manifestDirectoryPath +Invoke-Expression "npm i" +Invoke-Expression "npm run build" +Set-Location -Path $scriptDirectory + + +# Define the limit +$limitInt = 5 # Change this to your desired limit + +# Initialize an empty array to hold the strings +$ScenesToAnalyze = @() + +# Nested loops to generate the strings +for ($i = - $limitInt; $i -le $limitInt; $i++) { + for ($j = - $limitInt; $j -le $limitInt; $j++) { + Write-Host "Running for $i,$j" + Start-Process -FilePath "DCL_Pixyz.exe" -ArgumentList @("coords", "$i,$j", $outputDirectoryPath, $manifestDirectoryPath, "true", "false") -Wait + Write-Host "End running for $i,$j" + } +} diff --git a/consumer-server/src/components.ts b/consumer-server/src/components.ts index 0aeb6f99..688cf24b 100644 --- a/consumer-server/src/components.ts +++ b/consumer-server/src/components.ts @@ -8,7 +8,6 @@ import { AppComponents, GlobalContext } from './types' import { metricDeclarations } from './metrics' import { createSqsAdapter } from './adapters/sqs' import { createMessagesConsumerComponent } from './logic/message-consumer' -import { buildLicense } from './utils/license-builder' import { createLodGeneratorComponent } from './logic/lod-generator' import { createMessageHandlerComponent } from './logic/message-handler' import { createCloudStorageAdapter } from './adapters/storage' @@ -41,8 +40,6 @@ export async function initComponents(): Promise { const messageConsumer = await createMessagesConsumerComponent({ logs, queue, messageHandler }) - await buildLicense({ config, logs }) - return { config, logs, diff --git a/consumer-server/src/logic/lod-generator.ts b/consumer-server/src/logic/lod-generator.ts index 1a5e55d2..b6741815 100644 --- a/consumer-server/src/logic/lod-generator.ts +++ b/consumer-server/src/logic/lod-generator.ts @@ -23,8 +23,9 @@ export function createLodGeneratorComponent(): LodGeneratorComponent { outputPath: processOutput } - const commandToExecute = `${lodGeneratorProgram} "coords" "${basePointer}" "${outputPath}" "${sceneLodEntitiesManifestBuilder}"` - + + const commandToExecute = `${lodGeneratorProgram} "coords" "${basePointer}" "${outputPath}" "${sceneLodEntitiesManifestBuilder}" "false" "false"` + result = await new Promise((resolve, _) => { exec(commandToExecute, { timeout: 10 * 60 * 1000 }, (error, _, stderr) => { if (error) { diff --git a/consumer-server/src/utils/license-builder.ts b/consumer-server/src/utils/license-builder.ts deleted file mode 100644 index 7549e8bf..00000000 --- a/consumer-server/src/utils/license-builder.ts +++ /dev/null @@ -1,22 +0,0 @@ -import path from 'path' -import fs from 'fs' - -import { AppComponents } from '../types' - -export async function buildLicense({ config, logs }: Pick): Promise { - const logger = logs.getLogger('license-builder') - const licenseKey = (await config.getString('LODS_GENERATOR_LICENSE')) || '' // this is a SSM parameter pulled from AWS - const projectRoot = path.resolve(__dirname, '..', '..', '..') - - try { - const licenseKeyPath = path.resolve(projectRoot, 'pixyzsdk-29022024.lic') - const licenseKeyFile = fs.readFileSync(licenseKeyPath, 'utf8') - const replacedLicenseKey = licenseKeyFile.replace('{LICENSE_KEY}', licenseKey) - - fs.writeFileSync(licenseKeyPath, replacedLicenseKey, 'utf8') - logger.info('PiXYZ license built correctly') - } catch (error: any) { - logger.error('Could not build PiXYZ license', { error: error.message.replace(licenseKey, '****') || '' }) - throw error - } -} diff --git a/pixyzsdk-29022024.lic b/pixyzsdk-29022024.lic deleted file mode 100644 index f69a9000..00000000 --- a/pixyzsdk-29022024.lic +++ /dev/null @@ -1,70 +0,0 @@ - - - 1 - - - - - - - - - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {LICENSE_KEY} -