Simplify provider folders (#102)

* Use a single folder for all provider data
* Fixed mjpeg stream leak due to bug in Webkit where mjpeg stream not closed
* Improved upon the Android device setup

.gitignore

@@ -1,15 +1,14 @@
 WebDriverAgent
-logs/*
-build-context.tar
-apps/*
-ca-cert.pem
-ca-key.pem
-configs/supervision.p12
-minicap/*
 gads-project.log
+provider.log
 GADS
-gads-ui/*
-
-minicap-commands-ran.zip
-
-.vscode/launch.json
+hub/gads-ui/build/*
+hub/gads-ui/node_modules
+device_*
+gads-stream.apk
+.idea
+.DS_Store
+server.crt
+server.key
+jsconfig.json
+.vscode/launch.json

common/util/archive.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"GADS/provider/logger"
 	"archive/zip"
 	"bytes"
 	"fmt"
@@ -49,7 +50,7 @@ func UnzipInMemory(zipData []byte, dest string) error {
 		}
 	} else {
 		for _, f := range r.File {
-			fmt.Printf("Unzipping %s:\n", f.Name)
+			logger.ProviderLogger.LogDebug("unzip_app", fmt.Sprintf("Unzipping %s:\n", f.Name))
 			rc, err := f.Open()
 			if err != nil {
 				return err

docs/provider.md

@@ -25,14 +25,13 @@ Provider configuration is added through the GADS UI
 ## Provider data folder - optional
 The provider needs a persistent folder where logs, apps and other files might be stored.  
 
-You can skip this step and then starting the provider will look for `apps` and `logs` folders relative to the folder where the provider binary is located. 
-For example if you run the provider in `/Users/shamanec/Gads-provider` then it will look for `apps` and `logs` as `/Users/shamanec/Gads-provider/apps` and `/Users/shamanec/Gads-provider/logs` respectively.   
+You can skip this step and then starting the provider will create a folder named over the provider instance nickname relative to the folder where the provider binary is located. 
+For example if you run the provider in `/Users/shamanec/GADS` with nickname `Provider1` then it will create `/Users/shamanec/GADS/Provider1` folder respectively and store all related data there.  
 
 If you create a specific folder and provide it on startup - then the path will be relative to it.  
 Refer to the `--provider-folder` flag in [Running a provider instance](#running-a-provider-instance)
 
 1. Create a folder on your machine that will be accessible to the provider - name it any way you want.
-2. Open the newly created folder and inside create three more folders - `apps`, `logs`, `conf`
 
 ## Provider setup
 ### macOS
@@ -89,7 +88,7 @@ Refer to the `--provider-folder` flag in [Running a provider instance](#running-
 - Execute `./GADS provider` providing the following flags:  
   - `--nickname=` - mandatory, this is used to get the correct provider configuration from MongoDB
   - `--mongo-db=` - optional, IP address and port of the MongoDB instance (default is `localhost:27017`)
-  - `--provider-folder=` - optional, folder where provider should store logs and apps and get needed files for setup. Can be 1) relative path to the folder where provider binary is located or 2) full path on the host. Default is the folder where the binary is currently located, default is `.`
+  - `--provider-folder=` - optional, folder where provider should store logs and apps and other needed files. Can be relative path to the folder where provider binary is located or full path on the host - `./test`, `.`, `./test/test1`, `/Users/shamanec/Desktop/test` are all valid. Default is the folder where the binary is currently located - `.`
   - `--log-level=` - optional, how verbose should the provider logs be (default is `info`, use `debug` for more log output)
 
 ### Dependencies notes
@@ -138,7 +137,7 @@ This is an optional but a preferable step - it can make devices setup more auton
 - Set it up for supervision using a new(or existing) supervision identity. You can do that for free without having a paid MDM account.
 - Connect each consecutive device and supervise it using the same supervision identity.
 - Export your supervision identity file and choose a password.
-- Save your new supervision identity file in the `./conf` folder as `supervision.p12`.
+- Save your new supervision identity file in the provider folder as `supervision.p12`.
 
 **NB** You can skip supervising the devices and you should trust manually on first pair attempt by the provider but if devices are supervised in advance setup can be more autonomous.
 
@@ -182,17 +181,17 @@ You need a paid Apple Developer account to build and sign `WebDriverAgent` if yo
 Provider logs both to local files and to MongoDB.
 
 #### Provider logs
-Provider logs can be found in the `provider.log` file in the `/logs` folder relative to the supplied `provider-folder` flag on start.  
+Provider logs can be found in the `provider.log` file in the used provider folder - default or provided by the `--provider-folder` flag.  
 They will also be stored in MongoDB in DB `logs` and collection corresponding to the provider nickname.
 
 #### Device logs
-On start a log folder and file is created for each device in the `/logs` folder relative to the supplied `provider-folder` flag on start.  
+On start a log folder and file is created for each device relative to the used provider folder - default or provided by the `--provider-folder` flag.  
 They will also be stored in MongoDB in DB `logs` and collection corresponding to the device UDID.
 
 ### Additional notes
 #### Selenium Grid
 Devices can be automatically connected to Selenium Grid 4 instance. You need to create the Selenium Grid hub instance yourself and then setup the provider to connect to it.  
-To setup the provider download the Selenium server jar [release](https://github.com/SeleniumHQ/selenium/releases/tag/selenium-4.13.0) v4.13. Copy the downloaded jar and put it in the provider `./conf` folder.  
+To setup the provider download the Selenium server jar [release](https://github.com/SeleniumHQ/selenium/releases/tag/selenium-4.13.0) v4.13. Copy the downloaded jar and put it in the provider folder.  
 **NOTE** Currently versions above 4.13 don't work with Appium relay nodes and I haven't tested with lower versions. Use lower versions at your own risk.
 
 ### FAQ
@@ -211,6 +210,7 @@ To setup the provider download the Selenium server jar [release](https://github.
   - GADS uses `go-ios` to install and run WebDriverAgent on iOS < 17 devices on Linux/Windows
   - Make sure you've properly signed and created the [WebDriverAgent](#prepare-webdriveragent-file---linux-windows) ipa/app
   - Observe the provider logs - if installation is failing, you will see the full `go-ios` command used by GADS to install the prepared WebDriverAgent ipa/app. Copy the command and try to run it from terminal without the provider. Observe and debug the output.
+    - You might have to unlock your keychain, I haven't observed the need for this but it was reported - `security unlock-keychain -p yourMacPassword ~/Library/Keychains/login.keychain`
   - Observe the provider logs - if running of WDA is failing, you will see the full `go-ios` command used by GADS to run the installed WebDriverAgent. Copy the command and run it from terminal without the provider. Observe and debug the output
 - *[Android] I can load the device in the UI but there is no video, why?*
   - First option is that `GADS-android-stream` just doesn't work on your device - unlikely

hub/gads-ui/src/components/DeviceControl/StreamCanvas.js

@@ -1,5 +1,5 @@
 import { Auth } from "../../contexts/Auth"
-import { useContext } from "react"
+import {useContext, useEffect} from "react"
 import './StreamCanvas.css'
 import { Button, Divider, Grid, Stack } from "@mui/material"
 import HomeIcon from '@mui/icons-material/Home';
@@ -34,6 +34,12 @@ export default function StreamCanvas({ deviceData }) {
         streamUrl = `/device/${deviceData.udid}/android-stream-mjpeg`
     }
 
+    useEffect(() => {
+        return () => {
+            window.stop()
+        }
+    }, []);
+
     return (
         <div
             id='phone-imitation'

hub/hub.go

@@ -79,7 +79,6 @@ func StartHub(flags *pflag.FlagSet) {
 	defer db.MongoCtxCancel()
 
 	setLogging()
-	fmt.Println("")
 
 	err := db.AddAdminUserIfMissing()
 	if err != nil {

main.go

@@ -38,7 +38,7 @@ func main() {
 		},
 	}
 	providerCmd.Flags().String("nickname", "", "Nickname of the provider")
-	providerCmd.Flags().String("provider-folder", ".", "The folder where logs and apps are stored")
+	providerCmd.Flags().String("provider-folder", ".", "The folder where logs and other data will be stored")
 	providerCmd.Flags().String("log-level", "info", "The verbosity of the logs of the provider instance")
 	rootCmd.AddCommand(providerCmd)
 

provider/devices/android.go

@@ -32,7 +32,7 @@ func isGadsStreamServiceRunning(device *models.Device) (bool, error) {
 func installGadsStream(device *models.Device) error {
 	logger.ProviderLogger.LogInfo("android_device_setup", fmt.Sprintf("Installing GADS-stream apk on device `%v`", device.UDID))
 
-	cmd := exec.CommandContext(device.Context, "adb", "-s", device.UDID, "install", "-r", fmt.Sprintf("%s/conf/gads-stream.apk", config.Config.EnvConfig.ProviderFolder))
+	cmd := exec.CommandContext(device.Context, "adb", "-s", device.UDID, "install", "-r", fmt.Sprintf("%s/gads-stream.apk", config.Config.EnvConfig.ProviderFolder))
 	err := cmd.Run()
 	if err != nil {
 		return fmt.Errorf("installGadsStream: Error executing `%s` - %s", cmd.Args, err)
@@ -184,7 +184,7 @@ func uninstallAppAndroid(device *models.Device, packageName string) error {
 
 // Install app on Android device by apk name
 func installAppAndroid(device *models.Device, appName string) error {
-	cmd := exec.CommandContext(device.Context, "adb", "-s", device.UDID, "install", "-r", fmt.Sprintf("%s/apps/%s", config.Config.EnvConfig.ProviderFolder, appName))
+	cmd := exec.CommandContext(device.Context, "adb", "-s", device.UDID, "install", "-r", fmt.Sprintf("%s/%s", config.Config.EnvConfig.ProviderFolder, appName))
 
 	if err := cmd.Run(); err != nil {
 		device.Logger.LogError("install_app", fmt.Sprintf("installAppAndroid: Error executing `%s` trying to install app - %v", cmd.Args, err))

provider/devices/common.go

@@ -102,23 +102,23 @@ func updateDevices() {
 				db.AddCollectionIndex("appium_logs", newDevice.UDID, appiumCollectionIndexModel)
 
 				// Create logs directory for the device if it doesn't already exist
-				if _, err := os.Stat(fmt.Sprintf("%s/logs/device_%s", config.Config.EnvConfig.ProviderFolder, newDevice.UDID)); os.IsNotExist(err) {
-					err = os.Mkdir(fmt.Sprintf("%s/logs/device_%s", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), os.ModePerm)
+				if _, err := os.Stat(fmt.Sprintf("%s/device_%s", config.Config.EnvConfig.ProviderFolder, newDevice.UDID)); os.IsNotExist(err) {
+					err = os.Mkdir(fmt.Sprintf("%s/device_%s", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), os.ModePerm)
 					if err != nil {
 						logger.ProviderLogger.Errorf("updateDevices: Could not create logs folder for device `%s` - %s\n", newDevice.UDID, err)
 						continue
 					}
 				}
 
 				// Create a custom logger and attach it to the local device
-				deviceLogger, err := logger.CreateCustomLogger(fmt.Sprintf("%s/logs/device_%s/device.log", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), newDevice.UDID)
+				deviceLogger, err := logger.CreateCustomLogger(fmt.Sprintf("%s/device_%s/device.log", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), newDevice.UDID)
 				if err != nil {
 					logger.ProviderLogger.Errorf("updateDevices: Could not create custom logger for device `%s` - %s\n", newDevice.UDID, err)
 					continue
 				}
 				newDevice.Logger = *deviceLogger
 
-				appiumLogger, err := logger.NewAppiumLogger(fmt.Sprintf("%s/logs/device_%s/appium.log", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), newDevice.UDID)
+				appiumLogger, err := logger.NewAppiumLogger(fmt.Sprintf("%s/device_%s/appium.log", config.Config.EnvConfig.ProviderFolder, newDevice.UDID), newDevice.UDID)
 				if err != nil {
 					logger.ProviderLogger.Errorf("updateDevices: Could not create Appium logger for device `%s` - %s\n", newDevice.UDID, err)
 					continue
@@ -200,13 +200,6 @@ func setupAndroidDevice(device *models.Device) {
 		}
 	}
 
-	isStreamAvailable, err := isGadsStreamServiceRunning(device)
-	if err != nil {
-		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not check if GADS-stream is running on device `%v` - %v", device.UDID, err))
-		resetLocalDevice(device)
-		return
-	}
-
 	streamPort, err := providerutil.GetFreePort()
 	if err != nil {
 		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not allocate free host port for GADS-stream for device `%v` - %v", device.UDID, err))
@@ -215,44 +208,42 @@ func setupAndroidDevice(device *models.Device) {
 	}
 	device.StreamPort = streamPort
 
-	if !isStreamAvailable {
-		apps := getInstalledAppsAndroid(device)
-		if slices.Contains(apps, "com.shamanec.stream") {
-			err = uninstallGadsStream(device)
-			if err != nil {
-				logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not uninstall GADS-stream from Android device - %v:\n %v", device.UDID, err))
-				resetLocalDevice(device)
-				return
-			}
-			time.Sleep(1 * time.Second)
-		}
-
-		err = installGadsStream(device)
+	apps := getInstalledAppsAndroid(device)
+	if slices.Contains(apps, "com.shamanec.stream") {
+		err = uninstallGadsStream(device)
 		if err != nil {
-			logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not install GADS-stream on Android device - %v:\n %v", device.UDID, err))
+			logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not uninstall GADS-stream from Android device - %v:\n %v", device.UDID, err))
 			resetLocalDevice(device)
 			return
 		}
-		time.Sleep(1 * time.Second)
+		time.Sleep(3 * time.Second)
+	}
 
-		err = addGadsStreamRecordingPermissions(device)
-		if err != nil {
-			logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not set GADS-stream recording permissions on Android device - %v:\n %v", device.UDID, err))
-			resetLocalDevice(device)
-			return
-		}
-		time.Sleep(1 * time.Second)
+	err = installGadsStream(device)
+	if err != nil {
+		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not install GADS-stream on Android device - %v:\n %v", device.UDID, err))
+		resetLocalDevice(device)
+		return
+	}
+	time.Sleep(2 * time.Second)
 
-		err = startGadsStreamApp(device)
-		if err != nil {
-			logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not start GADS-stream app on Android device - %v:\n %v", device.UDID, err))
-			resetLocalDevice(device)
-			return
-		}
-		time.Sleep(1 * time.Second)
+	err = addGadsStreamRecordingPermissions(device)
+	if err != nil {
+		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not set GADS-stream recording permissions on Android device - %v:\n %v", device.UDID, err))
+		resetLocalDevice(device)
+		return
+	}
+	time.Sleep(1 * time.Second)
 
-		pressHomeButton(device)
+	err = startGadsStreamApp(device)
+	if err != nil {
+		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not start GADS-stream app on Android device - %v:\n %v", device.UDID, err))
+		resetLocalDevice(device)
+		return
 	}
+	time.Sleep(2 * time.Second)
+
+	pressHomeButton(device)
 
 	err = forwardGadsStream(device)
 	if err != nil {
@@ -394,7 +385,7 @@ func setupIOSDevice(device *models.Device) {
 
 	// If on Linux or Windows use the prebuilt and provided WebDriverAgent.ipa/app file
 	if config.Config.EnvConfig.OS != "darwin" {
-		wdaPath := fmt.Sprintf("%s/conf/%s", config.Config.EnvConfig.ProviderFolder, config.Config.EnvConfig.WebDriverBinary)
+		wdaPath := fmt.Sprintf("%s/%s", config.Config.EnvConfig.ProviderFolder, config.Config.EnvConfig.WebDriverBinary)
 		err = installAppWithPathIOS(device, wdaPath)
 		if err != nil {
 			logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Could not install WebDriverAgent on device `%s` - %s", device.UDID, err))
@@ -639,7 +630,7 @@ func createGridTOML(device *models.Device) error {
 		return fmt.Errorf("Failed marshalling TOML Appium config - %s", err)
 	}
 
-	file, err := os.Create(fmt.Sprintf("%s/conf/%s.toml", config.Config.EnvConfig.ProviderFolder, device.UDID))
+	file, err := os.Create(fmt.Sprintf("%s/%s.toml", config.Config.EnvConfig.ProviderFolder, device.UDID))
 	if err != nil {
 		return fmt.Errorf("Failed creating TOML Appium config file - %s", err)
 	}
@@ -658,12 +649,12 @@ func startGridNode(device *models.Device) {
 	cmd := exec.CommandContext(device.Context,
 		"java",
 		"-jar",
-		fmt.Sprintf("%s/conf/%s", config.Config.EnvConfig.ProviderFolder, config.Config.EnvConfig.SeleniumJarFile),
+		fmt.Sprintf("%s/%s", config.Config.EnvConfig.ProviderFolder, config.Config.EnvConfig.SeleniumJarFile),
 		"node",
 		"--host",
 		config.Config.EnvConfig.HostAddress,
 		"--config",
-		fmt.Sprintf("%s/conf/%s.toml", config.Config.EnvConfig.ProviderFolder, device.UDID),
+		fmt.Sprintf("%s/%s.toml", config.Config.EnvConfig.ProviderFolder, device.UDID),
 		"--grid-url",
 		config.Config.EnvConfig.SeleniumGrid,
 	)

provider/devices/ios.go

@@ -295,7 +295,7 @@ func mountDeveloperImageIOS(device *models.Device) error {
 func pairIOS(device *models.Device) error {
 	logger.ProviderLogger.LogInfo("ios_device_setup", fmt.Sprintf("Pairing device `%s`", device.UDID))
 
-	p12, err := os.ReadFile(fmt.Sprintf("%s/conf/supervision.p12", config.Config.EnvConfig.ProviderFolder))
+	p12, err := os.ReadFile(fmt.Sprintf("%s/supervision.p12", config.Config.EnvConfig.ProviderFolder))
 	if err != nil {
 		logger.ProviderLogger.LogWarn("ios_device_setup", fmt.Sprintf("Could not read supervision.p12 file when pairing device with UDID: %s, falling back to unsupervised pairing - %s", device.UDID, err))
 		err = ios.Pair(device.GoIOSDeviceEntry)
@@ -381,7 +381,7 @@ func installAppWithPathIOS(device *models.Device, path string) error {
 }
 
 func installAppIOS(device *models.Device, appName string) error {
-	appPath := fmt.Sprintf("%s/apps/%s", config.Config.EnvConfig.ProviderFolder, appName)
+	appPath := fmt.Sprintf("%s/%s", config.Config.EnvConfig.ProviderFolder, appName)
 	if config.Config.EnvConfig.OS == "darwin" {
 		cmd := exec.CommandContext(device.Context,
 			"xcrun",