Skip to content

Commit

Permalink
Remove resuming multipart uploads (#267)
Browse files Browse the repository at this point in the history 
ListParts is never fully complete in its output,
there is always a possibility that the List is
partial where another PutObjectPart() is being
uploaded in parallel.

While it may seem like ListParts and ListMultipartUploads
provide the possibility of resuming an upload, it would
be a mistake as these APIs were inherently never meant do
these things. This is one of the reasons why Amazon
recommends that one should avoid using these API calls to
complete or add parts to an upload - they instead
recommend to remember the parts being uploaded at the client.
  • Loading branch information
@harshavardhana
harshavardhana authored Jan 4, 2019
1 parent 794aa9a commit 3e89239
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 134 deletions.
4 changes: 2 additions & 2 deletions Docs/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,7 @@ __Parameters__
__Example__


The maximum size of a single object is limited to 5TB. putObject transparently uploads objects larger than 5MiB in multiple parts. This allows failed uploads to resume safely by only uploading the missing parts. Uploaded data is carefully verified using MD5SUM signatures.
The maximum size of a single object is limited to 5TB. putObject transparently uploads objects larger than 5MiB in multiple parts. Uploaded data is carefully verified using MD5SUM signatures.


```cs
Expand Down Expand Up @@ -864,7 +864,7 @@ __Parameters__
__Example__


The maximum size of a single object is limited to 5TB. putObject transparently uploads objects larger than 5MiB in multiple parts. This allows failed uploads to resume safely by only uploading the missing parts. Uploaded data is carefully verified using MD5SUM signatures.
The maximum size of a single object is limited to 5TB. putObject transparently uploads objects larger than 5MiB in multiple parts. Uploaded data is carefully verified using MD5SUM signatures.


```cs
Expand Down
72 changes: 11 additions & 61 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,13 +187,11 @@ public static void Main(string[] args)
// Test Putobject function
PutObject_Test1(minioClient).Wait();
PutObject_Test2(minioClient).Wait();

PutObject_Test3(minioClient).Wait();
PutObject_Test4(minioClient).Wait();
PutObject_Test5(minioClient).Wait();
PutObject_Test6(minioClient).Wait();
PutObject_Test7(minioClient).Wait();
PutObject_Test8(minioClient).Wait();

// Test StatObject function
StatObject_Test1(minioClient).Wait();
Expand Down Expand Up @@ -558,53 +556,6 @@ private async static Task PutObject_Test3(MinioClient minio)
}
}
private async static Task PutObject_Test4(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
string objectName = GetRandomName(10);
string contentType = "application/octet-stream";
Dictionary<string,string> args = new Dictionary<string,string>
{
{ "bucketName", bucketName},
{"objectName",objectName},
{"contentType", contentType},
{"data","1MB"},
{"size","4MB"},
};
try
{
// Putobject call with incorrect size of stream. See if PutObjectAsync call resumes
await Setup_Test(minio, bucketName);
using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB))
{
try
{
long size = 4 * MB;
long file_write_size = filestream.Length;

await minio.PutObjectAsync(bucketName,
objectName,
filestream,
size,
contentType);
}
catch (UnexpectedShortReadException)
{
// PutObject failed as expected since the stream size is incorrect
// default to actual stream size and complete the upload
await PutObject_Tester(minio, bucketName, objectName, null, contentType, 0, null,rsg.GenerateStreamFromSeed(1 * MB));
}
}
await TearDown(minio, bucketName);
new MintLogger("PutObject_Test4",putObjectSignature1,"Tests whether PutObject with incorrect stream-size passes",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log();
}
catch (Exception ex)
{
new MintLogger("PutObject_Test4",putObjectSignature1,"Tests whether PutObject with incorrect stream-size passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}
}

private async static Task PutObject_Test5(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
Expand Down Expand Up @@ -634,19 +585,18 @@ private async static Task PutObject_Test5(MinioClient minio)
Assert.IsTrue(statMeta.ContainsKey("x-amz-meta-customheader"));
Assert.IsTrue(statObject.metaData.ContainsKey("Content-Type") && statObject.metaData["Content-Type"].Equals("custom/contenttype"));
await TearDown(minio, bucketName);
new MintLogger("PutObject_Test5",putObjectSignature1,"Tests whether PutObject with different content-type passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
new MintLogger("PutObject_Test4",putObjectSignature1,"Tests whether PutObject with different content-type passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
}
catch (MinioException ex)
{
new MintLogger("PutObject_Test5",putObjectSignature1,"Tests whether PutObject with different content-type passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
new MintLogger("PutObject_Test4",putObjectSignature1,"Tests whether PutObject with different content-type passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}
if (!IsMintEnv())
{
File.Delete(fileName);
}
}

private async static Task PutObject_Test6(MinioClient minio)
private async static Task PutObject_Test5(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
Expand All @@ -663,14 +613,14 @@ private async static Task PutObject_Test6(MinioClient minio)
await Setup_Test(minio, bucketName);
await PutObject_Tester(minio, bucketName, objectName, null, null, 0, null, rsg.GenerateStreamFromSeed(1 * MB));
await TearDown(minio, bucketName);
new MintLogger("PutObject_Test6",putObjectSignature1,"Tests whether PutObject with no content-type passes for small object",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log();
new MintLogger("PutObject_Test5",putObjectSignature1,"Tests whether PutObject with no content-type passes for small object",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log();
}
catch (Exception ex)
{
new MintLogger("PutObject_Test6",putObjectSignature1,"Tests whether PutObject with no content-type passes for small object",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
new MintLogger("PutObject_Test5",putObjectSignature1,"Tests whether PutObject with no content-type passes for small object",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
}
}
private async static Task PutObject_Test7(MinioClient minio)
private async static Task PutObject_Test6(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
Expand All @@ -692,14 +642,14 @@ private async static Task PutObject_Test7(MinioClient minio)
await Task.WhenAll(tasks);
await minio.RemoveObjectAsync(bucketName, objectName);
await TearDown(minio, bucketName);
new MintLogger("PutObject_Test7",putObjectSignature1,"Tests thread safety of minioclient on a parallel put operation",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
new MintLogger("PutObject_Test6",putObjectSignature1,"Tests thread safety of minioclient on a parallel put operation",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
}
catch (Exception ex)
{
new MintLogger("PutObject_Test7",putObjectSignature1,"Tests thread safety of minioclient on a parallel put operation",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
new MintLogger("PutObject_Test6",putObjectSignature1,"Tests thread safety of minioclient on a parallel put operation",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
}
}
private async static Task PutObject_Test8(MinioClient minio)
private async static Task PutObject_Test7(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
Expand Down Expand Up @@ -731,11 +681,11 @@ await minio.PutObjectAsync(bucketName,
await minio.RemoveObjectAsync(bucketName, objectName);
await TearDown(minio, bucketName);
}
new MintLogger("PutObject_Test8",putObjectSignature1,"Tests whether PutObject with unknown stream-size passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
new MintLogger("PutObject_Test7",putObjectSignature1,"Tests whether PutObject with unknown stream-size passes",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
}
catch (Exception ex)
{
new MintLogger("PutObject_Test8",putObjectSignature1,"Tests whether PutObject with unknown stream-size passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
new MintLogger("PutObject_Test7",putObjectSignature1,"Tests whether PutObject with unknown stream-size passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(), args).Log();
}
}
private async static Task PutGetStatEncryptedObject_Test1(MinioClient minio)
Expand Down
Loading

0 comments on commit 3e89239

Please sign in to comment.