List dirs

alibaba.go

@@ -72,38 +72,48 @@ func NewAlibabaCloudOSSBackend(bucket string, prefix string, endpoint string, ss
 	return b
 }
 
+
 // ListObjects lists all objects in Alibaba Cloud OSS bucket, at prefix
 func (b AlibabaCloudOSSBackend) ListObjects(prefix string) ([]Object, error) {
-	var objects []Object
-
-	prefix = pathutil.Join(b.Prefix, prefix)
-	ossPrefix := oss.Prefix(prefix)
-	marker := oss.Marker("")
-	for {
-		lor, err := b.Bucket.ListObjects(oss.MaxKeys(50), marker, ossPrefix)
-		if err != nil {
-			return objects, err
-		}
-		for _, obj := range lor.Objects {
-			path := removePrefixFromObjectPath(prefix, obj.Key)
-			if objectPathIsInvalid(path) {
-				continue
+	return getObjects(b, prefix)
+}
+
+// ListFolders lists all folders in Alibaba Cloud OSS bucket, at prefix
+func (b AlibabaCloudOSSBackend) ListFolders(prefix string) ([]Folder, error) {
+	return getFolders(b, prefix)
+}
+
+func (b AlibabaCloudOSSBackend) ObjectIter(prefix string) <-chan Item {
+	ch := make(chan(Item))
+	go func() {
+		prefix = pathutil.Join(b.Prefix, prefix)
+		ossPrefix := oss.Prefix(prefix)
+		marker := oss.Marker("")
+		for {
+			lor, err := b.Bucket.ListObjects(oss.MaxKeys(50), marker, ossPrefix)
+			if err != nil {
+				ch <- Item{nil, err}
+				close(ch)
+				return
 			}
-			object := Object{
-				Path:         path,
-				Content:      []byte{},
-				LastModified: obj.LastModified,
+			for _, obj := range lor.Objects {
+				path := removePrefixFromObjectPath(prefix, obj.Key)
+				ch <- Item{
+					&Object{
+						Path:         path,
+						Content:      []byte{},
+						LastModified: obj.LastModified,
+					}, nil }
 			}
-			objects = append(objects, object)
-		}
-		if !lor.IsTruncated {
-			break
+			if !lor.IsTruncated {
+				break
+			}
+			ossPrefix = oss.Prefix(lor.Prefix)
+			marker = oss.Marker(lor.NextMarker)
 		}
-		ossPrefix = oss.Prefix(lor.Prefix)
-		marker = oss.Marker(lor.NextMarker)
-	}
-
-	return objects, nil
+		close(ch)
+	} ()
+	return ch
 }
 
 // GetObject retrieves an object from Alibaba Cloud OSS bucket, at prefix

alibaba_test.go

@@ -17,11 +17,11 @@ limitations under the License.
 package storage
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
-	"strconv"
 )
 
 type AlibabaTestSuite struct {
@@ -49,25 +49,39 @@ func (suite *AlibabaTestSuite) SetupSuite() {
 	path := "deleteme.txt"
 
 	for i := 0; i < testCount; i++ {
-		newPath := strconv.Itoa(i) + path
-		err := suite.NoPrefixAlibabaOSSBackend.PutObject(newPath, data)
+		testFilePath := fmt.Sprintf("%d%s", i, path)
+		testDirFilePath := fmt.Sprintf("testdir%d/%s", i, path)
+		err := suite.NoPrefixAlibabaOSSBackend.PutObject(testFilePath, data)
 		suite.Nil(err, "no error putting deleteme.txt using Alibaba Cloud OSS backend")
 
-		err = suite.SSEAlibabaOSSBackend.PutObject(newPath, data)
+		err = suite.NoPrefixAlibabaOSSBackend.PutObject(testDirFilePath, data)
+		suite.Nil(err, "no error putting testdir/deleteme.txt using Alibaba Cloud OSS backend")
+
+		err = suite.SSEAlibabaOSSBackend.PutObject(testFilePath, data)
 		suite.Nil(err, "no error putting deleteme.txt using Alibaba Cloud OSS backend (SSE)")
+
+		err = suite.SSEAlibabaOSSBackend.PutObject(testDirFilePath, data)
+		suite.Nil(err, "no error putting testdir/deleteme.txt using Alibaba Cloud OSS backend (SSE)")
 	}
 }
 
 func (suite *AlibabaTestSuite) TearDownSuite() {
 	path := "deleteme.txt"
 	for i := 0; i < testCount; i++ {
-		newPath := strconv.Itoa(i) + path
+		testFilePath := fmt.Sprintf("%d%s", i, path)
+		testDirFilePath := fmt.Sprintf("testdir%d/%s", i, path)
 
-		err := suite.NoPrefixAlibabaOSSBackend.DeleteObject(newPath)
+		err := suite.NoPrefixAlibabaOSSBackend.DeleteObject(testFilePath)
 		suite.Nil(err, "no error deleting deleteme.txt using AlibabaOSS backend")
 
-		err = suite.SSEAlibabaOSSBackend.DeleteObject(newPath)
+		err = suite.NoPrefixAlibabaOSSBackend.DeleteObject(testDirFilePath)
+		suite.Nil(err, "no error deleting testdir/deleteme.txt using AlibabaOSS backend")
+
+		err = suite.SSEAlibabaOSSBackend.DeleteObject(testFilePath)
 		suite.Nil(err, "no error deleting deleteme.txt using AlibabaOSS backend")
+
+		err = suite.SSEAlibabaOSSBackend.DeleteObject(testDirFilePath)
+		suite.Nil(err, "no error deleting testdir/deleteme.txt using AlibabaOSS backend")
 	}
 }
 
@@ -84,6 +98,19 @@ func (suite *AlibabaTestSuite) TestListObjects() {
 	suite.Equal(len(objs), testCount, "able to list objects")
 }
 
+func (suite *AlibabaTestSuite) TestListFolders() {
+	_, err := suite.BrokenAlibabaOSSBackend.ListFolders("")
+	suite.NotNil(err, "cannot list folders with bad bucket")
+
+	folders, err := suite.NoPrefixAlibabaOSSBackend.ListFolders("")
+	suite.Nil(err, "can list folders with good bucket, no prefix")
+	suite.Equal(len(folders), testCount, "able to list folders")
+
+	folders, err = suite.SSEAlibabaOSSBackend.ListFolders("")
+	suite.Nil(err, "can list objects with good bucket, SSE")
+	suite.Equal(len(folders), testCount, "able to list folders")
+}
+
 func (suite *AlibabaTestSuite) TestGetObject() {
 	_, err := suite.BrokenAlibabaOSSBackend.GetObject("this-file-cannot-possibly-exist.tgz")
 	suite.NotNil(err, "cannot get objects with bad bucket")

amazon.go

@@ -80,35 +80,46 @@ func NewAmazonS3BackendWithCredentials(bucket string, prefix string, region stri
 
 // ListObjects lists all objects in Amazon S3 bucket, at prefix
 func (b AmazonS3Backend) ListObjects(prefix string) ([]Object, error) {
-	var objects []Object
-	prefix = pathutil.Join(b.Prefix, prefix)
-	s3Input := &s3.ListObjectsInput{
-		Bucket: aws.String(b.Bucket),
-		Prefix: aws.String(prefix),
-	}
-	for {
-		s3Result, err := b.Client.ListObjects(s3Input)
-		if err != nil {
-			return objects, err
+	return getObjects(b, prefix)
+}
+
+// ListFolders lists all folders in Amazon S3 bucket, at prefix
+func (b AmazonS3Backend) ListFolders(prefix string) ([]Folder, error) {
+	return getFolders(b, prefix)
+}
+
+func (b AmazonS3Backend) ObjectIter(prefix string) <-chan Item {
+	ch := make(chan(Item))
+	go func() {
+		prefix = pathutil.Join(b.Prefix, prefix)
+		s3Input := &s3.ListObjectsInput{
+			Bucket: aws.String(b.Bucket),
+			Prefix: aws.String(prefix),
 		}
-		for _, obj := range s3Result.Contents {
-			path := removePrefixFromObjectPath(prefix, *obj.Key)
-			if objectPathIsInvalid(path) {
-				continue
+		for {
+			s3Result, err := b.Client.ListObjects(s3Input)
+			if err != nil {
+				ch <- Item{nil, err}
+				close(ch)
+				return
 			}
-			object := Object{
-				Path:         path,
-				Content:      []byte{},
-				LastModified: *obj.LastModified,
+			for _, obj := range s3Result.Contents {
+				path := removePrefixFromObjectPath(prefix, *obj.Key)
+				ch <- Item{
+					&Object{
+						Path:         path,
+						Content:      []byte{},
+						LastModified: *obj.LastModified,
+					}, nil }
 			}
-			objects = append(objects, object)
-		}
-		if !*s3Result.IsTruncated {
-			break
+			if !*s3Result.IsTruncated {
+				break
+			}
+			s3Input.Marker = s3Result.Contents[len(s3Result.Contents)-1].Key
 		}
-		s3Input.Marker = s3Result.Contents[len(s3Result.Contents)-1].Key
-	}
-	return objects, nil
+		close(ch)
+	} ()
+	return ch
 }
 
 // GetObject retrieves an object from Amazon S3 bucket, at prefix

amazon_test.go

@@ -44,12 +44,19 @@ func (suite *AmazonTestSuite) SetupSuite() {
 
 	data := []byte("some object")
 	path := "deleteme.txt"
+	dirFilePath := "testdir/deleteme.txt"
 
 	err := suite.NoPrefixAmazonS3Backend.PutObject(path, data)
 	suite.Nil(err, "no error putting deleteme.txt using AmazonS3 backend")
 
 	err = suite.SSEAmazonS3Backend.PutObject(path, data)
 	suite.Nil(err, "no error putting deleteme.txt using AmazonS3 backend (SSE)")
+
+	err = suite.NoPrefixAmazonS3Backend.PutObject(dirFilePath, data)
+	suite.Nil(err, "no error putting testdir/deleteme.txt using AmazonS3 backend")
+
+	err = suite.SSEAmazonS3Backend.PutObject(dirFilePath, data)
+	suite.Nil(err, "no error putting testdir/deleteme.txt using AmazonS3 backend (SSE)")
 }
 
 func (suite *AmazonTestSuite) TearDownSuite() {
@@ -58,17 +65,38 @@ func (suite *AmazonTestSuite) TearDownSuite() {
 
 	err = suite.SSEAmazonS3Backend.DeleteObject("deleteme.txt")
 	suite.Nil(err, "no error deleting deleteme.txt using AmazonS3 backend")
+
+	err = suite.NoPrefixAmazonS3Backend.DeleteObject("testdir/deleteme.txt")
+	suite.Nil(err, "no error deleting testdir/deleteme.txt using AmazonS3 backend")
+
+	err = suite.SSEAmazonS3Backend.DeleteObject("testdir/deleteme.txt")
+	suite.Nil(err, "no error deleting testdir/deleteme.txt using AmazonS3 backend")
 }
 
 func (suite *AmazonTestSuite) TestListObjects() {
 	_, err := suite.BrokenAmazonS3Backend.ListObjects("")
 	suite.NotNil(err, "cannot list objects with bad bucket")
 
-	_, err = suite.NoPrefixAmazonS3Backend.ListObjects("")
+	objs, err := suite.NoPrefixAmazonS3Backend.ListObjects("")
 	suite.Nil(err, "can list objects with good bucket, no prefix")
+	suite.Equal(len(objs), 1, "able to list objects")
 
-	_, err = suite.SSEAmazonS3Backend.ListObjects("")
+	objs, err = suite.SSEAmazonS3Backend.ListObjects("")
 	suite.Nil(err, "can list objects with good bucket, SSE")
+	suite.Equal(len(objs), 1, "able to list objects")
+}
+
+func (suite *AmazonTestSuite) TestListFolders() {
+	_, err := suite.BrokenAmazonS3Backend.ListFolders("")
+	suite.NotNil(err, "cannot list folders with bad bucket")
+
+	folders, err := suite.NoPrefixAmazonS3Backend.ListFolders("")
+	suite.Nil(err, "can list folders with good bucket, no prefix")
+	suite.Equal(len(folders), 1, "able to list folders")
+
+	folders, err = suite.SSEAmazonS3Backend.ListFolders("")
+	suite.Nil(err, "can list folders with good bucket, SSE")
+	suite.Equal(len(folders), 1, "able to list folders")
 }
 
 func (suite *AmazonTestSuite) TestGetObject() {

baidu.go

@@ -67,44 +67,53 @@ func NewBaiDuBOSBackend(bucket string, prefix string, endpoint string) *BaiduBOS
 
 // ListObjects lists all objects in Baidu Cloud BOS bucket, at prefix
 func (b BaiduBOSBackend) ListObjects(prefix string) ([]Object, error) {
-    var objects []Object
-
-    prefix = pathutil.Join(b.Prefix, prefix)
-    listObjectsArgs := &api.ListObjectsArgs{
-        Prefix:  prefix,
-        Marker:  "",
-        MaxKeys: 1000,
-    }
-    for {
-        lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs)
-        if err != nil {
-            return objects, err
-        }
-
-        for _, obj := range lor.Contents {
-            path := removePrefixFromObjectPath(prefix, obj.Key)
-            if objectPathIsInvalid(path) {
-                continue
-            }
-            lastModified, err := time.Parse(time.RFC3339, obj.LastModified)
-            if err != nil {
-                continue
-            }
-            object := Object{
-                Path:         path,
-                Content:      []byte{},
-                LastModified: lastModified,
-            }
-            objects = append(objects, object)
-        }
-        if !lor.IsTruncated {
-            break
-        }
-        listObjectsArgs.Prefix = lor.Prefix
-        listObjectsArgs.Marker = lor.NextMarker
-    }
-
-    return objects, nil
+	return getObjects(b, prefix)
+}
+
+// ListFolders lists all folders in Baidu Cloud BOS bucket, at prefix
+func (b BaiduBOSBackend) ListFolders(prefix string) ([]Folder, error) {
+	return getFolders(b, prefix)
+}
+
+func (b BaiduBOSBackend) ObjectIter(prefix string) <-chan Item {
+	ch := make(chan(Item))
+	go func() {
+		prefix = pathutil.Join(b.Prefix, prefix)
+		listObjectsArgs := &api.ListObjectsArgs{
+			Prefix:  prefix,
+			Marker:  "",
+			MaxKeys: 1000,
+		}
+		for {
+			lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs)
+			if err != nil {
+				ch <- Item{nil, err}
+				close(ch)
+				return
+			}
+
+			for _, obj := range lor.Contents {
+				path := removePrefixFromObjectPath(prefix, obj.Key)
+				lastModified, err := time.Parse(time.RFC3339, obj.LastModified)
+				if err != nil {
+					continue
+				}
+				ch <- Item{
+					&Object{
+						Path:		 path,
+						Content:	  []byte{},
+						LastModified: lastModified,
+					}, nil }
+			}
+			if !lor.IsTruncated {
+				break
+			}
+			listObjectsArgs.Prefix = lor.Prefix
+			listObjectsArgs.Marker = lor.NextMarker
+		}
+		close(ch)
+	} ()
+	return ch
 }
 
 // GetObject retrieves an object from Baidu Cloud BOS bucket, at prefix