Merge pull request #5 from taylorfinnell/start-implementing-client

introduce S3::Client
taylorfinnell · Nov 22, 2017 · 5cce439 · 5cce439
2 parents c8071b8 + 67bec6c
commit 5cce439
Show file tree

Hide file tree

Showing 36 changed files with 1,207 additions and 7 deletions.
diff --git a/examples/file_upload.cr b/examples/file_upload.cr
@@ -0,0 +1,19 @@
+require "../src/awscr-s3"
+require "secure_random"
+
+BUCKET = ENV["AWS_BUCKET"]
+KEY    = ENV["AWS_KEY"]
+SECRET = ENV["AWS_SECRET"]
+REGION = ENV["AWS_REGION"]
+
+client = Awscr::S3::Client.new(
+  region: REGION,
+  aws_access_key: KEY,
+  aws_secret_key: SECRET
+)
+
+uploader = Awscr::S3::FileUploader.new(client)
+
+File.open(File.expand_path("~/Downloads/EasyTomato_0.8.trx"), "r") do |file|
+  puts uploader.upload(BUCKET, "someobjectkey", file)
+end
diff --git a/examples/list_objects.cr b/examples/list_objects.cr
@@ -0,0 +1,16 @@
+require "../src/awscr-s3"
+
+BUCKET = ENV["AWS_BUCKET"]
+KEY    = ENV["AWS_KEY"]
+SECRET = ENV["AWS_SECRET"]
+REGION = ENV["AWS_REGION"]
+
+client = Awscr::S3::Client.new(
+  region: REGION,
+  aws_access_key: KEY,
+  aws_secret_key: SECRET
+)
+
+client.list_objects(bucket: BUCKET, max_keys: 10).each do |response|
+  p response.contents.map(&.key)
+end
diff --git a/examples/multipart_upload.cr b/examples/multipart_upload.cr
@@ -0,0 +1,37 @@
+require "../src/awscr-s3"
+require "secure_random"
+
+BUCKET = ENV["AWS_BUCKET"]
+KEY    = ENV["AWS_KEY"]
+SECRET = ENV["AWS_SECRET"]
+REGION = ENV["AWS_REGION"]
+
+client = Awscr::S3::Client.new(
+  region: REGION,
+  aws_access_key: KEY,
+  aws_secret_key: SECRET
+)
+
+object = SecureRandom.uuid
+
+upload = client.start_multipart_upload(
+  bucket: BUCKET,
+  object: object
+)
+
+uploaded_part = client.upload_part(
+  bucket: BUCKET,
+  object: object,
+  upload_id: upload.upload_id,
+  part_number: 1,
+  part: IO::Memory.new("A" * 5000001) # 5mb min
+)
+
+final = client.complete_multipart_upload(
+  bucket: BUCKET,
+  object: object,
+  upload_id: upload.upload_id,
+  parts: [uploaded_part]
+)
+
+p final.inspect
diff --git a/examples/object_put_get.cr b/examples/object_put_get.cr
@@ -0,0 +1,20 @@
+require "../src/awscr-s3"
+require "secure_random"
+
+BUCKET = ENV["AWS_BUCKET"]
+KEY    = ENV["AWS_KEY"]
+SECRET = ENV["AWS_SECRET"]
+REGION = ENV["AWS_REGION"]
+
+client = Awscr::S3::Client.new(
+  region: REGION,
+  aws_access_key: KEY,
+  aws_secret_key: SECRET
+)
+
+object = SecureRandom.uuid
+
+client.put_object(bucket: BUCKET, object: object, body: IO::Memory.new("Hey!"))
+
+resp = client.get_object(bucket: BUCKET, object: object)
+puts resp.body
diff --git a/shard.yml b/shard.yml
@@ -1,5 +1,5 @@
 name: awscr-s3
-version: 0.1.1
+version: 0.1.2
 
 authors:
   - Taylor Finnell <[email protected]>
@@ -16,3 +16,7 @@ dependencies:
 development_dependencies:
   timecop:
     github: waterlink/timecop.cr
+    branch: master
+  webmock:
+    github: manastech/webmock.cr
+    branch: master
diff --git a/spec/awscr-s3/client_spec.cr b/spec/awscr-s3/client_spec.cr
@@ -0,0 +1,281 @@
+require "../spec_helper"
+
+module Awscr::S3
+  describe Client do
+    describe "abort_multipart_upload" do
+      it "aborts an upload" do
+        WebMock.stub(:delete, "http://s3.amazonaws.com/bucket/object?uploadId=upload_id")
+               .to_return(status: 204)
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.abort_multipart_upload("bucket", "object", "upload_id")
+
+        result.should be_true
+      end
+    end
+
+    describe "start_multipart_upload" do
+      it "starts a multipart upload" do
+        resp = <<-RESP
+          <?xml version="1.0" encoding="UTF-8"?>
+          <InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+            <Bucket>bucket</Bucket>
+            <Key>object</Key>
+            <UploadId>FxtGq8otGhDtYJa5VYLpPOBQo2niM2a1YR8wgcwqHJ1F1Djflj339mEfpm7NbYOoIg.6bIPeXl2RB82LuAnUkTQUEz_ReIu2wOwawGc0Z4SLERxoXospqANXDazuDmRF</UploadId>
+          </InitiateMultipartUploadResult>
+        RESP
+
+        WebMock.stub(:post, "http://s3.amazonaws.com/bucket/object?uploads")
+               .to_return(status: 200, body: resp)
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.start_multipart_upload("bucket", "object")
+
+        result.should eq(Response::StartMultipartUpload.new(
+          "bucket",
+          "object",
+          "FxtGq8otGhDtYJa5VYLpPOBQo2niM2a1YR8wgcwqHJ1F1Djflj339mEfpm7NbYOoIg.6bIPeXl2RB82LuAnUkTQUEz_ReIu2wOwawGc0Z4SLERxoXospqANXDazuDmRF"
+        ))
+      end
+    end
+
+    describe "upload_part" do
+      it "uploads a part" do
+        WebMock.stub(:put, "http://s3.amazonaws.com/bucket2/obj2?partNumber=1&uploadId=123")
+               .with(body: "test")
+               .to_return(status: 200, body: "", headers: {"ETag" => "etag"})
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.upload_part("bucket2", "obj2", "123", 1, "test")
+
+        result.should eq(
+          Response::UploadPartOutput.new("etag", 1, "123")
+        )
+      end
+    end
+
+    describe "complete_multipart_upload" do
+      it "completes a multipart upload" do
+        post_body = "<?xml version=\"1.0\"?>\n<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>etag</ETag></Part></CompleteMultipartUpload>\n"
+
+        resp_body = <<-RESP_BODY
+        <?xml version="1.0" encoding="UTF-8"?>
+          <CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+          <Location>http://s3.amazonaws.com/screensnapr-development/test</Location>
+          <Bucket>screensnapr-development</Bucket>
+          <Key>test</Key>
+          <ETag>&quot;7611c6414e4b58f22ff9f59a2c1767b7-2&quot;</ETag>
+        </CompleteMultipartUploadResult>
+        RESP_BODY
+
+        WebMock.stub(:post, "http://s3.amazonaws.com/bucket/object?uploadId=upload_id")
+               .with(body: post_body)
+               .to_return(status: 200, body: resp_body)
+
+        outputs = [Response::UploadPartOutput.new("etag", 1, "upload_id")]
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.complete_multipart_upload("bucket", "object", "upload_id", outputs)
+
+        result.should eq(
+          Response::CompleteMultipartUpload.new(
+            "http://s3.amazonaws.com/screensnapr-development/test",
+            "test",
+            "\"7611c6414e4b58f22ff9f59a2c1767b7-2\""
+          )
+        )
+      end
+    end
+
+    describe "delete_object" do
+      it "returns true if object deleted" do
+        WebMock.stub(:delete, "http://s3.amazonaws.com/blah/obj?")
+               .to_return(status: 204)
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.delete_object("blah", "obj")
+
+        result.should be_true
+      end
+    end
+
+    describe "put_object" do
+      it "can do a basic put" do
+        io = IO::Memory.new("Hello")
+
+        WebMock.stub(:put, "http://s3.amazonaws.com/mybucket/object.txt")
+               .with(body: "Hello")
+               .to_return(body: "", headers: {"ETag" => "etag"})
+
+        client = Client.new("us-east-1", "key", "secret")
+        resp = client.put_object("mybucket", "object.txt", io)
+
+        resp.should eq(Response::PutObjectOutput.new("etag"))
+      end
+    end
+
+    describe "list_objects" do
+      it "handles pagination" do
+        resp = <<-RESP
+        <?xml version="1.0" encoding="UTF-8"?>
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+          <Name>bucket</Name>
+          <Prefix/>
+          <KeyCount>1</KeyCount>
+          <MaxKeys>1</MaxKeys>
+          <IsTruncated>true</IsTruncated>
+          <NextContinuationToken>token</NextContinuationToken
+          <Contents>
+              <Key>my-image.jpg</Key>
+              <LastModified>2009-10-12T17:50:30.000Z</LastModified>
+              <ETag>"fba9dede5f27731c9771645a39863328"</ETag>
+              <Size>434234</Size>
+              <StorageClass>STANDARD</StorageClass>
+          </Contents>
+        </ListBucketResult>
+        RESP
+
+        resp2 = <<-RESP
+        <?xml version="1.0" encoding="UTF-8"?>
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+          <Name>bucket</Name>
+          <Prefix/>
+          <KeyCount>1</KeyCount>
+          <MaxKeys>1</MaxKeys>
+          <IsTruncated>false</IsTruncated>
+          <Contents>
+              <Key>key2</Key>
+              <LastModified>2009-10-12T17:50:30.000Z</LastModified>
+              <ETag>"fba9dede5f27731c9771645a39863329"</ETag>
+              <Size>1337</Size>
+              <StorageClass>STANDARD</StorageClass>
+          </Contents>
+        </ListBucketResult>
+        RESP
+
+        WebMock.stub(:get, "http://s3.amazonaws.com/bucket?list-type=2&max-keys=1&continuation-token=token")
+               .to_return(body: resp2)
+
+        WebMock.stub(:get, "http://s3.amazonaws.com/bucket?list-type=2&max-keys=1")
+               .to_return(body: resp)
+
+        client = Client.new("us-east-1", "key", "secret")
+
+        objects = [] of Response::ListObjectsV2
+        client.list_objects("bucket", max_keys: 1).each do |output|
+          objects << output
+        end
+
+        expected_objects = [
+          Object.new("my-image.jpg", 434234,
+            "\"fba9dede5f27731c9771645a39863328\""),
+          Object.new("key2", 1337,
+            "\"fba9dede5f27731c9771645a39863329\""),
+        ]
+
+        objects.should eq([
+          Response::ListObjectsV2.new("bucket", "", 1, 1, true, "token",
+            [expected_objects.first]),
+          Response::ListObjectsV2.new("bucket", "", 1, 1, false, "",
+            [expected_objects.last]),
+        ])
+      end
+
+      it "supports basic case" do
+        resp = <<-RESP
+        <?xml version="1.0" encoding="UTF-8"?>
+        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+          <Name>blah</Name>
+          <Prefix/>
+          <KeyCount>205</KeyCount>
+          <MaxKeys>1000</MaxKeys>
+          <IsTruncated>false</IsTruncated>
+          <Contents>
+              <Key>my-image.jpg</Key>
+              <LastModified>2009-10-12T17:50:30.000Z</LastModified>
+              <ETag>&quot;fba9dede5f27731c9771645a39863328&quot;</ETag>
+              <Size>434234</Size>
+              <StorageClass>STANDARD</StorageClass>
+          </Contents>
+          <Contents>
+              <Key>key2</Key>
+              <LastModified>2009-10-12T17:50:30.000Z</LastModified>
+              <ETag>&quot;fba9dede5f27731c9771645a39863329&quot;</ETag>
+              <Size>1337</Size>
+              <StorageClass>STANDARD</StorageClass>
+          </Contents>
+        </ListBucketResult>
+        RESP
+
+        WebMock.stub(:get, "http://s3.amazonaws.com/blah?list-type=2")
+               .to_return(body: resp)
+
+        expected_objects = [
+          Object.new("my-image.jpg", 434234,
+            "\"fba9dede5f27731c9771645a39863328\""),
+          Object.new("key2", 1337,
+            "\"fba9dede5f27731c9771645a39863329\""),
+        ]
+
+        client = Client.new("us-east-1", "key", "secret")
+
+        objs = client.list_objects("blah").each do |output|
+          output.should eq(Response::ListObjectsV2.new("blah", "", 205, 1000, false, "", expected_objects))
+        end
+      end
+    end
+
+    describe "list_buckets" do
+      it "returns buckets on success" do
+        resp = <<-RESP
+        <?xml version="1.0" encoding="UTF-8"?>
+        <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+          <Owner>
+            <ID>bcaf1ffd86f461ca5fb16fd081034f</ID>
+            <DisplayName>webfile</DisplayName>
+          </Owner>
+          <Buckets>
+            <Bucket>
+              <Name>quotes</Name>
+              <CreationDate>2006-02-03T16:45:09.000Z</CreationDate>
+            </Bucket>
+          </Buckets>
+        </ListAllMyBucketsResult>
+        RESP
+
+        WebMock.stub(:get, "http://s3.amazonaws.com/?")
+               .to_return(body: resp)
+
+        client = Client.new("us-east-1", "key", "secret")
+        output = client.list_buckets
+
+        output.should eq(Response::ListAllMyBuckets.new([
+          Bucket.new("quotes", "2006-02-03T16:45:09.000Z"),
+        ]))
+      end
+    end
+
+    describe "head_bucket" do
+      it "raises if bucket does not exist" do
+        WebMock.stub(:head, "http://s3.amazonaws.com/blah2?")
+               .to_return(status: 404)
+
+        client = Client.new("us-east-1", "key", "secret")
+
+        expect_raises do
+          client.head_bucket("blah2")
+        end
+      end
+
+      it "returns true if bucket exists" do
+        WebMock.stub(:head, "http://s3.amazonaws.com/blah?")
+               .to_return(status: 200)
+
+        client = Client.new("us-east-1", "key", "secret")
+        result = client.head_bucket("blah")
+
+        result.should be_true
+      end
+    end
+  end
+end