Config: Add versioning (#1671)

* Config: Version Management

* Engine: Improve visibility of TestConfigAllJsonResponse failures

* Config: Update cmd/config to allow upgrades

* Config: Add Version2 to rename GDAX

* Config: Restructure versioning to share types

This restructure allows us to share types between versions, avoids
needing to import the versions, and puts the test fixtures in same
package.
It's a win on all fronts

* Config: Fix SetNTPCheck using log

Called from engine before logger is inited, and also just wrong to use
log to communicate with user

* Config: Improve TestMigrateConfig

* Config: Drop requirement for versions to be registered in sequence

Checking the versions at Deploy is much saner.

* Config: Fix file encrypted but flag not set

* Config: Add -edit and encryption upgrade to cmd/config

This simplifies the handling for encryption prompts by moving it to a
field on config, allowing us to simplify all the places were were
passing around config

Also moves password entry to being secure (echo-off)

* Tests: Fix inconsistent should/must assertions

cmd/config/config.go

@@ -1,82 +1,177 @@
 package main
 
 import (
-	"encoding/json"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
 	"os"
+	"slices"
+	"strings"
 
+	"github.com/buger/jsonparser"
 	"github.com/thrasher-corp/gocryptotrader/common/file"
 	"github.com/thrasher-corp/gocryptotrader/config"
 )
 
-// EncryptOrDecrypt returns a string from a boolean
-func EncryptOrDecrypt(encrypt bool) string {
-	if encrypt {
-		return "encrypted"
-	}
-	return "decrypted"
-}
+var commands = []string{"upgrade", "encrypt", "decrypt"}
 
 func main() {
-	var inFile, outFile, key string
-	var encrypt bool
+	fmt.Println("GoCryptoTrader: config-helper tool")
+
 	defaultCfgFile := config.DefaultFilePath()
-	flag.StringVar(&inFile, "infile", defaultCfgFile, "The config input file to process.")
-	flag.StringVar(&outFile, "outfile", defaultCfgFile+".out", "The config output file.")
-	flag.BoolVar(&encrypt, "encrypt", true, "Whether to encrypt or decrypt.")
-	flag.StringVar(&key, "key", "", "The key to use for AES encryption.")
-	flag.Parse()
-
-	log.Println("GoCryptoTrader: config-helper tool.")
-
-	if key == "" {
-		result, err := config.PromptForConfigKey(false)
-		if err != nil {
-			log.Fatalf("Unable to obtain encryption/decryption key: %s", err)
-		}
-		key = string(result)
+
+	var in, out, keyStr string
+	var inplace bool
+
+	fs := flag.NewFlagSet("config", flag.ExitOnError)
+	fs.Usage = func() { usage(fs) }
+	fs.StringVar(&in, "in", defaultCfgFile, "The config input file to process")
+	fs.StringVar(&out, "out", "[in].out", "The config output file")
+	fs.BoolVar(&inplace, "edit", false, "Edit; Save result to the original file")
+	fs.StringVar(&keyStr, "key", "", "The key to use for AES encryption")
+
+	cmd, args := parseCommand(os.Args[1:])
+	if cmd == "" {
+		usage(fs)
+		os.Exit(2)
+	}
+
+	if err := fs.Parse(args); err != nil {
+		fatal(err.Error())
+	}
+
+	if inplace {
+		out = in
+	} else if out == "[in].out" {
+		out = in + ".out"
+	}
+
+	key := []byte(keyStr)
+	var err error
+	switch cmd {
+	case "upgrade":
+		err = upgradeFile(in, out, key)
+	case "decrypt":
+		err = encryptWrapper(in, out, key, false, decryptFile)
+	case "encrypt":
+		err = encryptWrapper(in, out, key, true, encryptFile)
 	}
 
-	fileData, err := os.ReadFile(inFile)
 	if err != nil {
-		log.Fatalf("Unable to read input file %s. Error: %s.", inFile, err)
+		fatal(err.Error())
 	}
 
-	if config.ConfirmECS(fileData) && encrypt {
-		log.Println("File is already encrypted. Decrypting..")
-		encrypt = false
+	fmt.Println("Success! File written to " + out)
+}
+
+func upgradeFile(in, out string, key []byte) error {
+	c := &config.Config{
+		EncryptionKeyProvider: func(_ bool) ([]byte, error) {
+			if len(key) != 0 {
+				return key, nil
+			}
+			return config.PromptForConfigKey(false)
+		},
 	}
 
-	if !config.ConfirmECS(fileData) && !encrypt {
-		var result interface{}
-		errf := json.Unmarshal(fileData, &result)
-		if errf != nil {
-			log.Fatal(errf)
-		}
-		log.Println("File is already decrypted. Encrypting..")
-		encrypt = true
+	if err := c.ReadConfigFromFile(in, true); err != nil {
+		return err
 	}
 
-	var data []byte
-	if encrypt {
-		data, err = config.EncryptConfigFile(fileData, []byte(key))
-		if err != nil {
-			log.Fatalf("Unable to encrypt config data. Error: %s.", err)
-		}
-	} else {
-		data, err = config.DecryptConfigFile(fileData, []byte(key))
-		if err != nil {
-			log.Fatalf("Unable to decrypt config data. Error: %s.", err)
+	return c.SaveConfigToFile(out)
+}
+
+type encryptFunc func(string, []byte) ([]byte, error)
+
+func encryptWrapper(in, out string, key []byte, confirmKey bool, fn encryptFunc) error {
+	if len(key) == 0 {
+		var err error
+		if key, err = config.PromptForConfigKey(confirmKey); err != nil {
+			return err
 		}
 	}
+	outData, err := fn(in, key)
+	if err != nil {
+		return err
+	}
+	if err := file.Write(out, outData); err != nil {
+		return fmt.Errorf("unable to write output file %s; Error: %w", out, err)
+	}
+	return nil
+}
+
+func encryptFile(in string, key []byte) ([]byte, error) {
+	if config.IsFileEncrypted(in) {
+		return nil, errors.New("file is already encrypted")
+	}
+	outData, err := config.EncryptConfigFile(readFile(in), key)
+	if err != nil {
+		return nil, fmt.Errorf("unable to encrypt config data. Error: %w", err)
+	}
+	return outData, nil
+}
+
+func decryptFile(in string, key []byte) ([]byte, error) {
+	if !config.IsFileEncrypted(in) {
+		return nil, errors.New("file is already decrypted")
+	}
+	outData, err := config.DecryptConfigFile(readFile(in), key)
+	if err != nil {
+		return nil, fmt.Errorf("unable to decrypt config data. Error: %w", err)
+	}
+	if outData, err = jsonparser.Set(outData, []byte("-1"), "encryptConfig"); err != nil {
+		return nil, fmt.Errorf("unable to decrypt config data. Error: %w", err)
+	}
+	return outData, nil
+}
 
-	err = file.Write(outFile, data)
+func readFile(in string) []byte {
+	fileData, err := os.ReadFile(in)
 	if err != nil {
-		log.Fatalf("Unable to write output file %s. Error: %s", outFile, err)
+		fatal("Unable to read input file " + in + "; Error: " + err.Error())
 	}
-	log.Printf(
-		"Successfully %s input file %s and wrote output to %s.\n",
-		EncryptOrDecrypt(encrypt), inFile, outFile,
-	)
+	return fileData
+}
+
+func fatal(msg string) {
+	fmt.Fprintln(os.Stderr, msg)
+	os.Exit(2)
+}
+
+// parseCommand will return the single non-flag parameter from os.Args, and return the remaining args
+// If none is provided, too many, usage() will be called and exit 1
+func parseCommand(a []string) (cmd string, args []string) {
+	cmds, rem := []string{}, []string{}
+	for _, s := range a {
+		if slices.Contains(commands, s) {
+			cmds = append(cmds, s)
+		} else {
+			rem = append(rem, s)
+		}
+	}
+	switch len(cmds) {
+	case 0:
+		fmt.Fprintln(os.Stderr, "No command provided")
+	case 1: //
+		return cmds[0], rem
+	default:
+		fmt.Fprintln(os.Stderr, "Too many commands provided: "+strings.Join(cmds, ", "))
+	}
+	return "", nil
+}
+
+// usage prints command usage and exits 1
+func usage(fs *flag.FlagSet) {
+	//nolint:dupword // deliberate duplication  of commands
+	fmt.Fprintln(os.Stderr, `
+Usage:
+config [arguments] <command>
+
+The commands are:
+	encrypt 	encrypt infile and write to outfile
+	decrypt 	decrypt infile and write to outfile
+	upgrade 	upgrade the version of a decrypted config file
+
+The arguments are:`)
+	fs.PrintDefaults()
 }

common/common.go

@@ -69,6 +69,8 @@ var (
 	ErrNoResponse             = errors.New("no response")
 	ErrTypeAssertFailure      = errors.New("type assert failure")
 	ErrUnknownError           = errors.New("unknown error")
+	ErrGettingField           = errors.New("error getting field")
+	ErrSettingField           = errors.New("error setting field")
 )
 
 var (