From 1038b62e1c1cb399762041499ec7bcf1dd121450 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 6 Jan 2025 09:00:32 -0800 Subject: [PATCH] Replace random input generator (#357) Replace data generator Sideeffects: * Remove CSV source (never used) * Remove select benchmark (not really useful and select is deprecated by AWS anyway) * Upgrade pkg to v3 --- api/api.go | 2 +- cli/analyze.go | 2 +- cli/benchclient.go | 2 +- cli/benchmark.go | 2 +- cli/cli.go | 7 +- cli/client.go | 6 +- cli/clientmode.go | 2 +- cli/cmp.go | 2 +- cli/delete.go | 2 +- cli/fanout.go | 2 +- cli/flags.go | 2 +- cli/generator.go | 22 ----- cli/get.go | 2 +- cli/list.go | 2 +- cli/merge.go | 2 +- cli/mixed.go | 2 +- cli/multipart.go | 2 +- cli/print.go | 2 +- cli/progress-bar.go | 2 +- cli/put.go | 2 +- cli/retention.go | 2 +- cli/rlimit.go | 2 +- cli/select.go | 100 ------------------- cli/snowball.go | 2 +- cli/stat.go | 2 +- cli/versioned.go | 2 +- cli/zip.go | 2 +- go.mod | 10 +- go.sum | 17 ++-- pkg/bench/benchmark.go | 2 +- pkg/bench/collector.go | 2 +- pkg/bench/delete.go | 2 +- pkg/bench/get.go | 2 +- pkg/bench/list.go | 2 +- pkg/bench/mixed.go | 2 +- pkg/bench/multipart.go | 2 +- pkg/bench/ops.go | 2 +- pkg/bench/retention.go | 2 +- pkg/bench/s3zip.go | 2 +- pkg/bench/select.go | 2 +- pkg/bench/stat.go | 2 +- pkg/bench/versioned.go | 2 +- pkg/generator/circular.go | 121 ----------------------- pkg/generator/csv.go | 166 -------------------------------- pkg/generator/generator_test.go | 64 +----------- pkg/generator/options.go | 2 - pkg/generator/random.go | 21 ++-- pkg/generator/scambler.go | 126 ------------------------ 48 files changed, 68 insertions(+), 666 deletions(-) delete mode 100644 cli/select.go delete mode 100644 pkg/generator/circular.go delete mode 100644 pkg/generator/csv.go delete mode 100644 pkg/generator/scambler.go diff --git a/api/api.go b/api/api.go index f0edbe3c..1f0868f1 100644 --- a/api/api.go +++ b/api/api.go @@ -27,7 +27,7 @@ import ( "time" "github.com/klauspost/compress/zstd" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/aggregate" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/analyze.go b/cli/analyze.go index 1954251a..0cf755b1 100644 --- a/cli/analyze.go +++ b/cli/analyze.go @@ -32,7 +32,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/api" "github.com/minio/warp/pkg/aggregate" "github.com/minio/warp/pkg/bench" diff --git a/cli/benchclient.go b/cli/benchclient.go index 80ef0a7d..86933c95 100644 --- a/cli/benchclient.go +++ b/cli/benchclient.go @@ -27,7 +27,7 @@ import ( "time" "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" "github.com/minio/websocket" ) diff --git a/cli/benchmark.go b/cli/benchmark.go index 46ef82f5..a9dbd0ab 100644 --- a/cli/benchmark.go +++ b/cli/benchmark.go @@ -34,7 +34,7 @@ import ( "github.com/minio/cli" "github.com/minio/madmin-go/v3" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/api" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/cli.go b/cli/cli.go index 8decd149..4b2c0c33 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -31,9 +31,9 @@ import ( "github.com/cheggaaa/pb" "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" - "github.com/minio/pkg/v2/trie" - "github.com/minio/pkg/v2/words" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/trie" + "github.com/minio/pkg/v3/words" "github.com/minio/warp/pkg" completeinstall "github.com/posener/complete/cmd/install" ) @@ -91,7 +91,6 @@ func init() { deleteCmd, listCmd, statCmd, - selectCmd, versionedCmd, retentionCmd, multipartCmd, diff --git a/cli/client.go b/cli/client.go index 4b097861..53afde13 100644 --- a/cli/client.go +++ b/cli/client.go @@ -38,9 +38,9 @@ import ( md5simd "github.com/minio/md5-simd" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/minio/pkg/v2/certs" - "github.com/minio/pkg/v2/console" - "github.com/minio/pkg/v2/ellipses" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/ellipses" "github.com/minio/warp/pkg" "golang.org/x/net/http2" ) diff --git a/cli/clientmode.go b/cli/clientmode.go index e10fac04..14efc3ea 100644 --- a/cli/clientmode.go +++ b/cli/clientmode.go @@ -24,7 +24,7 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" ) var clientFlags = []cli.Flag{} diff --git a/cli/cmp.go b/cli/cmp.go index 29ed5ba4..c042d0b1 100644 --- a/cli/cmp.go +++ b/cli/cmp.go @@ -26,7 +26,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/delete.go b/cli/delete.go index a0839228..bdebf4fd 100644 --- a/cli/delete.go +++ b/cli/delete.go @@ -19,7 +19,7 @@ package cli import ( "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/fanout.go b/cli/fanout.go index 889ddb59..36552dcf 100644 --- a/cli/fanout.go +++ b/cli/fanout.go @@ -19,7 +19,7 @@ package cli import ( "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/flags.go b/cli/flags.go index b30b6a06..af9eefb0 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -24,7 +24,7 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" "github.com/minio/warp/pkg/generator" diff --git a/cli/generator.go b/cli/generator.go index 7dfb3d22..31d2c5f9 100644 --- a/cli/generator.go +++ b/cli/generator.go @@ -43,26 +43,6 @@ var genFlags = []cli.Flag{ }, } -func newGenSourceCSV(ctx *cli.Context) func() generator.Source { - prefixSize := 8 - if ctx.Bool("noprefix") { - prefixSize = 0 - } - - g := generator.WithCSV().Size(25, 1000) - - size, err := toSize(ctx.String("obj.size")) - fatalIf(probe.NewError(err), "Invalid obj.size specified") - src, err := generator.NewFn(g.Apply(), - generator.WithCustomPrefix(ctx.String("prefix")), - generator.WithPrefixSize(prefixSize), - generator.WithSize(int64(size)), - generator.WithRandomSize(ctx.Bool("obj.randsize")), - ) - fatalIf(probe.NewError(err), "Unable to create data generator") - return src -} - // newGenSource returns a new generator func newGenSource(ctx *cli.Context, sizeField string) func() generator.Source { prefixSize := 8 @@ -74,8 +54,6 @@ func newGenSource(ctx *cli.Context, sizeField string) func() generator.Source { switch ctx.String("obj.generator") { case "random": g = generator.WithRandomData() - case "csv": - g = generator.WithCSV().Size(25, 1000) default: err := errors.New("unknown generator type:" + ctx.String("obj.generator")) fatal(probe.NewError(err), "Invalid -generator parameter") diff --git a/cli/get.go b/cli/get.go index 4256ba64..09eccbe9 100644 --- a/cli/get.go +++ b/cli/get.go @@ -20,7 +20,7 @@ package cli import ( "github.com/minio/cli" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/list.go b/cli/list.go index 5341f4ee..9a632418 100644 --- a/cli/list.go +++ b/cli/list.go @@ -19,7 +19,7 @@ package cli import ( "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/merge.go b/cli/merge.go index 4a544f68..4edff681 100644 --- a/cli/merge.go +++ b/cli/merge.go @@ -26,7 +26,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/minio/cli" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/mixed.go b/cli/mixed.go index 237ac8ac..9c476288 100644 --- a/cli/mixed.go +++ b/cli/mixed.go @@ -23,7 +23,7 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/probe" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/multipart.go b/cli/multipart.go index d8f8c55b..17773ecc 100644 --- a/cli/multipart.go +++ b/cli/multipart.go @@ -22,7 +22,7 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/print.go b/cli/print.go index 85ad0eff..f9e2d146 100644 --- a/cli/print.go +++ b/cli/print.go @@ -26,7 +26,7 @@ import ( "github.com/cheggaaa/pb" "github.com/minio/mc/pkg/probe" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" ) // causeMessage container for golang error messages diff --git a/cli/progress-bar.go b/cli/progress-bar.go index b5ddae6c..5795758e 100644 --- a/cli/progress-bar.go +++ b/cli/progress-bar.go @@ -24,7 +24,7 @@ import ( "github.com/cheggaaa/pb" "github.com/fatih/color" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" ) // progress extender. diff --git a/cli/put.go b/cli/put.go index c424014c..1ae68d63 100644 --- a/cli/put.go +++ b/cli/put.go @@ -24,7 +24,7 @@ import ( "github.com/minio/cli" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/retention.go b/cli/retention.go index 9cea5c77..816e680e 100644 --- a/cli/retention.go +++ b/cli/retention.go @@ -19,7 +19,7 @@ package cli import ( "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/rlimit.go b/cli/rlimit.go index c5ac549c..5a60d641 100644 --- a/cli/rlimit.go +++ b/cli/rlimit.go @@ -20,7 +20,7 @@ package cli import ( "runtime/debug" - "github.com/minio/pkg/v2/sys" + "github.com/minio/pkg/v3/sys" ) func setMaxResources() (err error) { diff --git a/cli/select.go b/cli/select.go deleted file mode 100644 index 9879213f..00000000 --- a/cli/select.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Warp (C) 2023 MinIO, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package cli - -import ( - "github.com/minio/cli" - "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" - "github.com/minio/warp/pkg/bench" -) - -var selectFlags = []cli.Flag{ - cli.IntFlag{ - Name: "objects", - Value: 2500, - Usage: "Number of objects to upload.", - }, - cli.StringFlag{ - Name: "obj.size", - Value: "10MiB", - Usage: "Size of each generated object. Can be a number or 10KiB/MiB/GiB. All sizes are base 2 binary.", - }, - cli.StringFlag{ - Name: "query", - Value: "select * from s3object", - Usage: "select query expression", - }, -} - -var selectCmd = cli.Command{ - Name: "select", - Usage: "benchmark select objects", - Action: mainSelect, - Before: setGlobalsFromContext, - Flags: combineFlags(globalFlags, ioFlags, selectFlags, genFlags, benchFlags, analyzeFlags), - CustomHelpTemplate: `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} [FLAGS] - -> see https://github.com/minio/warp - -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}`, -} - -// mainSelect is the entry point for select command. -func mainSelect(ctx *cli.Context) error { - checkSelectSyntax(ctx) - sse := newSSE(ctx) - b := bench.Select{ - Common: getCommon(ctx, newGenSourceCSV(ctx)), - CreateObjects: ctx.Int("objects"), - SelectOpts: minio.SelectObjectOptions{ - Expression: ctx.String("query"), - ExpressionType: minio.QueryExpressionTypeSQL, - // Set any encryption headers - ServerSideEncryption: sse, - // TODO: support all variations including, json/parquet - InputSerialization: minio.SelectObjectInputSerialization{ - CSV: &minio.CSVInputOptions{ - RecordDelimiter: "\n", - FieldDelimiter: ",", - FileHeaderInfo: minio.CSVFileHeaderInfoUse, - }, - }, - OutputSerialization: minio.SelectObjectOutputSerialization{ - CSV: &minio.CSVOutputOptions{ - RecordDelimiter: "\n", - FieldDelimiter: ",", - }, - }, - }, - } - return runBench(ctx, &b) -} - -func checkSelectSyntax(ctx *cli.Context) { - if ctx.Int("objects") < 1 { - console.Fatal("At least one object must be tested") - } - checkAnalyze(ctx) - checkBenchmark(ctx) -} diff --git a/cli/snowball.go b/cli/snowball.go index 733653dd..3ff61e0e 100644 --- a/cli/snowball.go +++ b/cli/snowball.go @@ -20,7 +20,7 @@ package cli import ( "github.com/minio/cli" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/stat.go b/cli/stat.go index a0278d9c..5b36f55d 100644 --- a/cli/stat.go +++ b/cli/stat.go @@ -20,7 +20,7 @@ package cli import ( "github.com/minio/cli" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/versioned.go b/cli/versioned.go index 4cba3e44..b7a14cf4 100644 --- a/cli/versioned.go +++ b/cli/versioned.go @@ -23,7 +23,7 @@ import ( "github.com/minio/cli" "github.com/minio/mc/pkg/probe" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/cli/zip.go b/cli/zip.go index 01b04506..55ae8149 100644 --- a/cli/zip.go +++ b/cli/zip.go @@ -22,7 +22,7 @@ import ( "time" "github.com/minio/cli" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/bench" ) diff --git a/go.mod b/go.mod index 9d4f2771..c55fb34b 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,9 @@ require ( github.com/minio/mc v0.0.0-20250102131134-b1a3941f22f5 github.com/minio/md5-simd v1.1.2 github.com/minio/minio-go/v7 v7.0.83 - github.com/minio/pkg/v2 v2.0.19 + github.com/minio/pkg/v3 v3.0.26-0.20250106155027-2becdc33e233 github.com/minio/websocket v1.6.0 github.com/posener/complete v1.2.3 - github.com/secure-io/sio-go v0.3.1 golang.org/x/net v0.33.0 golang.org/x/time v0.9.0 gopkg.in/yaml.v3 v3.0.1 @@ -37,11 +36,11 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect - github.com/lestrrat-go/jwx v1.2.30 // indirect + github.com/lestrrat-go/jwx/v2 v2.1.3 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -51,7 +50,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.61.0 // indirect @@ -62,6 +60,8 @@ require ( github.com/rjeczalik/notify v0.9.3 // indirect github.com/rs/xid v1.6.0 // indirect github.com/safchain/ethtool v0.5.9 // indirect + github.com/secure-io/sio-go v0.3.1 // indirect + github.com/segmentio/asm v1.2.0 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tinylib/msgp v1.2.5 // indirect diff --git a/go.sum b/go.sum index f36501de..8986788a 100644 --- a/go.sum +++ b/go.sum @@ -55,17 +55,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= +github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx v1.2.30 h1:VKIFrmjYn0z2J51iLPadqoHIVLzvWNa1kCsTqNDHYPA= -github.com/lestrrat-go/jwx v1.2.30/go.mod h1:vMxrwFhunGZ3qddmfmEm2+uced8MSI6QFWGTKygjSzQ= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= +github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= @@ -95,8 +94,8 @@ github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0 github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= -github.com/minio/pkg/v2 v2.0.19 h1:r187/k/oVH9H0DDwvLY5WipkJaZ4CLd4KI3KgIUExR0= -github.com/minio/pkg/v2 v2.0.19/go.mod h1:luK9LAhQlAPzSuF6F326XSCKjMc1G3Tbh+a9JYwqh8M= +github.com/minio/pkg/v3 v3.0.26-0.20250106155027-2becdc33e233 h1:SR5q/92Xqkj2Zg3O3sPnJvms+/ZMN5W1mSiA3htdnhc= +github.com/minio/pkg/v3 v3.0.26-0.20250106155027-2becdc33e233/go.mod h1:mIaN552nu0D2jiSk5BQC8LB25f44ytbOBJCuLtksX7Q= github.com/minio/websocket v1.6.0 h1:CPvnQvNvlVaQmvw5gtJNyYQhg4+xRmrPNhBbv8BdpAE= github.com/minio/websocket v1.6.0/go.mod h1:COH1CePZfHT9Ec1O7vZjTlX5uEPpyYnrifPNbu665DM= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -105,8 +104,6 @@ github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmt github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -137,6 +134,8 @@ github.com/safchain/ethtool v0.5.9 h1://6RvaOKFf3nQ0rl5+8zBbE4/72455VC9Jq61pfq67 github.com/safchain/ethtool v0.5.9/go.mod h1:w8oSsZeowyRaM7xJJBAbubzzrOkwO8TBgPSEqPP/5mg= github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= diff --git a/pkg/bench/benchmark.go b/pkg/bench/benchmark.go index cdb02e96..29cc593e 100644 --- a/pkg/bench/benchmark.go +++ b/pkg/bench/benchmark.go @@ -27,7 +27,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" "golang.org/x/time/rate" diff --git a/pkg/bench/collector.go b/pkg/bench/collector.go index 508df949..5b48a914 100644 --- a/pkg/bench/collector.go +++ b/pkg/bench/collector.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" ) type Collector struct { diff --git a/pkg/bench/delete.go b/pkg/bench/delete.go index 7d7bee14..3e0313e8 100644 --- a/pkg/bench/delete.go +++ b/pkg/bench/delete.go @@ -26,7 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/get.go b/pkg/bench/get.go index 589e26fb..7ad13b4c 100644 --- a/pkg/bench/get.go +++ b/pkg/bench/get.go @@ -27,7 +27,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/list.go b/pkg/bench/list.go index 8c229447..7e9f4006 100644 --- a/pkg/bench/list.go +++ b/pkg/bench/list.go @@ -27,7 +27,7 @@ import ( "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/mixed.go b/pkg/bench/mixed.go index 440347d6..7c8c472a 100644 --- a/pkg/bench/mixed.go +++ b/pkg/bench/mixed.go @@ -28,7 +28,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/multipart.go b/pkg/bench/multipart.go index 6dade97f..f1d24211 100644 --- a/pkg/bench/multipart.go +++ b/pkg/bench/multipart.go @@ -29,7 +29,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/ops.go b/pkg/bench/ops.go index e0361229..7e8141e9 100644 --- a/pkg/bench/ops.go +++ b/pkg/bench/ops.go @@ -29,7 +29,7 @@ import ( "time" "github.com/dustin/go-humanize" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" ) type Operations []Operation diff --git a/pkg/bench/retention.go b/pkg/bench/retention.go index 3786be5b..e6abf683 100644 --- a/pkg/bench/retention.go +++ b/pkg/bench/retention.go @@ -26,7 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/s3zip.go b/pkg/bench/s3zip.go index 72730807..4d7997c5 100644 --- a/pkg/bench/s3zip.go +++ b/pkg/bench/s3zip.go @@ -29,7 +29,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/select.go b/pkg/bench/select.go index 155434a3..f1adc9ea 100644 --- a/pkg/bench/select.go +++ b/pkg/bench/select.go @@ -27,7 +27,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/stat.go b/pkg/bench/stat.go index 519ac35e..edab0fd3 100644 --- a/pkg/bench/stat.go +++ b/pkg/bench/stat.go @@ -26,7 +26,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/bench/versioned.go b/pkg/bench/versioned.go index 11d24114..ee928d5f 100644 --- a/pkg/bench/versioned.go +++ b/pkg/bench/versioned.go @@ -28,7 +28,7 @@ import ( "time" "github.com/minio/minio-go/v7" - "github.com/minio/pkg/v2/console" + "github.com/minio/pkg/v3/console" "github.com/minio/warp/pkg/generator" ) diff --git a/pkg/generator/circular.go b/pkg/generator/circular.go deleted file mode 100644 index df91fd03..00000000 --- a/pkg/generator/circular.go +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Warp (C) 2019-2020 MinIO, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package generator - -import ( - "errors" - "io" -) - -type circularBuffer struct { - data []byte - // left aliases the data at the current read position. - left []byte - - // The total number of bytes to return - // When this - want int64 - read int64 -} - -// Reset will reset the circular buffer. -// The number of bytes to return can be specified. -// If the number of bytes wanted is <= 0 the value will not be updated. -func (c *circularBuffer) Reset(want int64) io.ReadSeeker { - if want > 0 { - c.want = want - } - c.read = 0 - c.left = c.data - return c -} - -// Implement seeker compatible circular buffer, -// implemented for minio-go to allow retries. -func (c *circularBuffer) Seek(offset int64, whence int) (n int64, err error) { - // Switch through whence. - switch whence { - default: - return 0, errors.New("circularBuffer.Seek: invalid whence") - case io.SeekStart: - if offset > c.want { - return 0, io.EOF - } - c.read = offset - case io.SeekCurrent: - if offset+c.read > c.want { - return 0, io.EOF - } - c.read += offset - case io.SeekEnd: - if offset > 0 { - return 0, io.EOF - } - if c.want+offset < 0 { - return 0, io.ErrShortBuffer - } - c.read = c.want + offset - } - if c.read < 0 { - return 0, errors.New("circularBuffer.Seek: negative position") - } - return c.read, nil -} - -// newCircularBuffer returns a new circular buffer. -// Data will be served -func newCircularBuffer(data []byte, size int64) *circularBuffer { - return &circularBuffer{ - data: data, - left: data, - want: size, - read: 0, - } -} - -func (c *circularBuffer) Read(p []byte) (n int, err error) { - if len(c.data) == 0 { - return 0, errors.New("circularBuffer: no data") - } - for len(p) > 0 { - if len(c.left) == 0 { - c.left = c.data - } - remain := c.want - c.read - if remain <= 0 { - if remain != 0 { - panic(remain) - } - return n, io.EOF - } - - // Make sure we don't overread. - toDo := c.left - if int64(len(toDo)) > remain { - toDo = toDo[:remain] - } - - copied := copy(p, toDo) - // Assign remaining back to c.left - c.left = toDo[copied:] - p = p[copied:] - c.read += int64(copied) - n += copied - } - return n, nil -} diff --git a/pkg/generator/csv.go b/pkg/generator/csv.go deleted file mode 100644 index ca4e3975..00000000 --- a/pkg/generator/csv.go +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Warp (C) 2019-2020 MinIO, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package generator - -import ( - "errors" - "fmt" - "math/rand" -) - -// WithCSV returns default CSV Opts -func WithCSV() CsvOpts { - return csvOptsDefaults() -} - -// Apply applies all the opts for CSVOpts -func (o CsvOpts) Apply() Option { - return func(opts *Options) error { - if err := o.validate(); err != nil { - return err - } - opts.csv = o - opts.src = newCsv - return nil - } -} - -func (o CsvOpts) validate() error { - if o.rows < 0 { - return errors.New("csv: rows <= 0") - } - if o.cols < 0 { - return errors.New("csv: cols <= 0") - } - if o.minLen > o.maxLen { - return fmt.Errorf("WithCSV.FieldLen: min:%d > max:%d", o.minLen, o.maxLen) - } - - return nil -} - -// Size sets the size of generated CSV. -func (o CsvOpts) Size(cols, rows int) CsvOpts { - o.rows = rows - o.cols = cols - return o -} - -// Comma sets the comma character. Only ASCII values should be used. -func (o CsvOpts) Comma(c byte) CsvOpts { - o.comma = c - return o -} - -// FieldLen sets the length of each field. -func (o CsvOpts) FieldLen(minSize, maxSize int) CsvOpts { - o.minLen = minSize - o.maxLen = maxSize - return o -} - -// RngSeed will which to a fixed RNG seed to make usage predictable. -func (o CsvOpts) RngSeed(s int64) CsvOpts { - o.seed = &s - return o -} - -// CsvOpts provides options for CSV generation. -type CsvOpts struct { - err error - seed *int64 - cols, rows int - minLen, maxLen int - comma byte -} - -func csvOptsDefaults() CsvOpts { - return CsvOpts{ - err: nil, - cols: 15, - rows: 1000, - comma: ',', - seed: nil, - minLen: 5, - maxLen: 15, - } -} - -type csvSource struct { - buf *circularBuffer - - // We may need a faster RNG for this... - rng *rand.Rand - obj Object - - o Options - builder []byte -} - -func newCsv(o Options) (Source, error) { - c := csvSource{ - o: o, - } - c.builder = make([]byte, 0, o.csv.maxLen+1) - c.buf = newCircularBuffer(make([]byte, o.csv.maxLen*(o.csv.cols+1)*(o.csv.rows+1)), o.totalSize) - rndSrc := rand.NewSource(int64(rand.Uint64())) - if o.csv.seed != nil { - rndSrc = rand.NewSource(*o.csv.seed) - } - c.rng = rand.New(rndSrc) - c.obj.ContentType = "text/csv" - c.obj.Size = 0 - c.obj.setPrefix(o) - - return &c, nil -} - -func (c *csvSource) Object() *Object { - opts := c.o.csv - dst := c.buf.data[:0] - c.obj.Size = c.o.getSize(c.rng) - for i := 0; i < opts.rows; i++ { - for j := 0; j < opts.cols; j++ { - fieldLen := 1 + opts.minLen - if opts.minLen != opts.maxLen { - fieldLen += c.rng.Intn(opts.maxLen - opts.minLen) - } - build := c.builder[:fieldLen] - randASCIIBytes(build[:fieldLen-1], c.rng) - build[fieldLen-1] = opts.comma - if j == opts.cols-1 { - build[fieldLen-1] = '\n' - } - dst = append(dst, build...) - } - } - c.buf.data = dst - c.obj.Reader = c.buf.Reset(0) - var nBuf [16]byte - randASCIIBytes(nBuf[:], c.rng) - c.obj.setName(string(nBuf[:]) + ".csv") - return &c.obj -} - -func (c *csvSource) String() string { - return fmt.Sprintf("CSV data. %d columns, %d rows.", c.o.csv.cols, c.o.csv.rows) -} - -func (c *csvSource) Prefix() string { - return c.obj.Prefix -} diff --git a/pkg/generator/generator_test.go b/pkg/generator/generator_test.go index 153e35e2..7bddefdb 100644 --- a/pkg/generator/generator_test.go +++ b/pkg/generator/generator_test.go @@ -41,14 +41,6 @@ func TestNew(t *testing.T) { wantErr: false, wantSize: 1 << 20, }, - { - name: "CSV", - args: args{ - opts: []Option{WithCSV().Apply()}, - }, - wantErr: false, - wantSize: 1 << 20, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -110,9 +102,9 @@ func TestNew(t *testing.T) { t.Errorf("New() size = %v, wantSize = %v", len(b), tt.wantSize) return } - n, err = obj.Reader.Seek(10, 1) - if err != io.EOF { - t.Errorf("Expected io.EOF, got %v", err) + n, err = obj.Reader.Seek(10, io.SeekCurrent) + if err != io.ErrUnexpectedEOF { + t.Errorf("Expected io.ErrUnexpectedEOF, got %v", err) return } if n != 0 { @@ -123,56 +115,6 @@ func TestNew(t *testing.T) { } } -func BenchmarkWithCSV(b *testing.B) { - type args struct { - opts []Option - } - tests := []struct { - name string - args args - }{ - { - name: "64KiB-5x100", - args: args{opts: []Option{WithSize(1 << 16), WithCSV().Size(5, 100).Apply()}}, - }, - { - name: "1MiB-10x500", - args: args{opts: []Option{WithSize(1 << 20), WithCSV().Size(10, 500).Apply()}}, - }, - { - name: "10MiB-50x1000", - args: args{opts: []Option{WithSize(10 << 20), WithCSV().Size(50, 1000).Apply()}}, - }, - } - for _, tt := range tests { - b.Run(tt.name, func(b *testing.B) { - got, err := New(tt.args.opts...) - if err != nil { - b.Errorf("New() error = %v", err) - return - } - obj := got.Object() - payload, err := io.ReadAll(obj.Reader) - if err != nil { - b.Errorf("ioutil error = %v", err) - return - } - b.SetBytes(int64(len(payload))) - // ioutil.WriteFile(tt.name+".csv", payload, os.ModePerm) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - obj := got.Object() - _, err := io.Copy(io.Discard, obj.Reader) - if err != nil { - b.Errorf("New() error = %v", err) - return - } - } - }) - } -} - func BenchmarkWithRandomData(b *testing.B) { type args struct { opts []Option diff --git a/pkg/generator/options.go b/pkg/generator/options.go index 2673c9cf..377d0258 100644 --- a/pkg/generator/options.go +++ b/pkg/generator/options.go @@ -30,7 +30,6 @@ type Options struct { src func(o Options) (Source, error) customPrefix string random RandomOpts - csv CsvOpts minSize int64 totalSize int64 randomPrefix int @@ -61,7 +60,6 @@ func defaultOptions() Options { o := Options{ src: newRandom, totalSize: 1 << 20, - csv: csvOptsDefaults(), random: randomOptsDefaults(), randomPrefix: 0, } diff --git a/pkg/generator/random.go b/pkg/generator/random.go index 5a20dc40..8398c250 100644 --- a/pkg/generator/random.go +++ b/pkg/generator/random.go @@ -20,9 +20,10 @@ package generator import ( "errors" "fmt" - "io" "math/rand" "sync/atomic" + + "github.com/minio/pkg/v3/rng" ) func WithRandomData() RandomOpts { @@ -76,7 +77,7 @@ func randomOptsDefaults() RandomOpts { } type randomSrc struct { - buf *scrambler + source *rng.Reader rng *rand.Rand obj Object o Options @@ -88,7 +89,6 @@ func newRandom(o Options) (Source, error) { if o.random.seed != nil { rndSrc = rand.NewSource(*o.random.seed) } - rng := rand.New(rndSrc) size := o.random.size if int64(size) > o.totalSize { @@ -98,16 +98,14 @@ func newRandom(o Options) (Source, error) { return nil, fmt.Errorf("size must be >= 0, got %d", size) } - // Seed with random data. - data := make([]byte, size) - _, err := io.ReadFull(rng, data) + input, err := rng.NewReader(rng.WithRNG(rand.New(rndSrc)), rng.WithSize(o.totalSize)) if err != nil { return nil, err } r := randomSrc{ - o: o, - rng: rng, - buf: newScrambler(data, o.totalSize, rng), + o: o, + rng: rand.New(rndSrc), + source: input, obj: Object{ Reader: nil, Name: "", @@ -127,7 +125,8 @@ func (r *randomSrc) Object() *Object { r.obj.setName(fmt.Sprintf("%d.%s.rnd", atomic.LoadUint64(&r.counter), string(nBuf[:]))) // Reset scrambler - r.obj.Reader = r.buf.Reset(r.obj.Size) + r.source.ResetSize(r.obj.Size) + r.obj.Reader = r.source return &r.obj } @@ -135,7 +134,7 @@ func (r *randomSrc) String() string { if r.o.randSize { return fmt.Sprintf("Random data; random size up to %d bytes", r.o.totalSize) } - return fmt.Sprintf("Random data; %d bytes total", r.buf.want) + return fmt.Sprintf("Random data; %d bytes total", r.o.totalSize) } func (r *randomSrc) Prefix() string { diff --git a/pkg/generator/scambler.go b/pkg/generator/scambler.go deleted file mode 100644 index 2d8881fc..00000000 --- a/pkg/generator/scambler.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Warp (C) 2021 MinIO, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package generator - -import ( - "crypto/aes" - "crypto/cipher" - "errors" - "io" - "math" - "math/rand" - - "github.com/secure-io/sio-go" -) - -type scrambler struct { - // Data source - stream *sio.EncReader - // The total number of bytes to return - want int64 - // Number of bytes read - read int64 -} - -// Reset will reset the scrambler. -// The number of bytes to return can be specified. -// If the number of bytes wanted is <= 0 the value will not be updated. -func (c *scrambler) Reset(want int64) io.ReadSeeker { - if want > 0 { - c.want = want - } - c.read = 0 - return c -} - -// Implement seeker compatible circular buffer, -// implemented for minio-go to allow retries. -func (c *scrambler) Seek(offset int64, whence int) (n int64, err error) { - // Switch through whence. - switch whence { - default: - return 0, errors.New("circularBuffer.Seek: invalid whence") - case io.SeekStart: - if offset > c.want { - return 0, io.EOF - } - c.read = offset - case io.SeekCurrent: - if offset+c.read > c.want { - return 0, io.EOF - } - c.read += offset - case io.SeekEnd: - if offset > 0 { - return 0, io.EOF - } - if c.want+offset < 0 { - return 0, io.ErrShortBuffer - } - c.read = c.want + offset - } - if c.read < 0 { - return 0, errors.New("circularBuffer.Seek: negative position") - } - return c.read, nil -} - -// newCircularBuffer a reader that will produce (virtually) infinitely amounts of random data. -func newScrambler(data []byte, size int64, rng *rand.Rand) *scrambler { - var randSrc [16]byte - - _, err := io.ReadFull(rng, randSrc[:]) - if err != nil { - panic(err) - } - rand.New(rng).Read(randSrc[:]) - block, _ := aes.NewCipher(randSrc[:]) - gcm, _ := cipher.NewGCM(block) - stream := sio.NewStream(gcm, sio.BufSize) - - return &scrambler{ - want: size, - read: 0, - stream: stream.EncryptReader(newCircularBuffer(data, math.MaxInt64), randSrc[:stream.NonceSize()], nil), - } -} - -func (c *scrambler) Read(p []byte) (n int, err error) { - remain := c.want - c.read - if remain <= 0 { - if remain != 0 { - panic(remain) - } - return n, io.EOF - } - // Make sure we don't overread. - toDo := len(p) - if int64(toDo) > remain { - p = p[:remain] - } - copied, err := io.ReadFull(c.stream, p) - if err != nil { - return copied, err - } - // Assign remaining back to c.left - c.read += int64(copied) - if c.read == c.want { - return copied, io.EOF - } - return copied, nil -}