Skip to content

File upload performance improvement #1321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

File upload performance improvement #1321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
15da77e
perf: Use MemoryPool<byte> to reduce memory allocations.
loyldg Jul 17, 2025
8e8282b
perf: Memory allocation optimization
loyldg Jul 17, 2025
ab6532f
fix: Add missing IsAmazonChinaEndPoint check
loyldg Jul 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 12 additions & 20 deletions Minio/ApiEndpoints/ObjectOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
* limitations under the License.
*/

using System.Buffers;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Net;
Expand Down Expand Up @@ -576,8 +577,10 @@ public async Task<PutObjectResponse> PutObjectAsync(PutObjectArgs args,
if ((args.ObjectSize < Constants.MinimumPartSize || isSnowball) && args.ObjectSize >= 0 &&
args.ObjectStreamData is not null)
{
var bytes = await ReadFullAsync(args.ObjectStreamData, (int)args.ObjectSize).ConfigureAwait(false);
var bytesRead = bytes.Length;
using var tempMemory = MemoryPool<byte>.Shared.Rent((int)args.ObjectSize);
var bytesRead = await ReadFullAsync(args.ObjectStreamData, tempMemory.Memory)
.ConfigureAwait(false);
var bytes = tempMemory.Memory[..bytesRead];
if (bytesRead != (int)args.ObjectSize)
throw new UnexpectedShortReadException(
$"Data read {bytesRead.ToString(CultureInfo.InvariantCulture)} is shorter than the size {args.ObjectSize.ToString(CultureInfo.InvariantCulture)} of input buffer.");
Expand Down Expand Up @@ -916,9 +919,11 @@ private async Task<IDictionary<int, string>> PutObjectPartAsync(PutObjectPartArg
var etags = new Dictionary<int, string>();
var progressReport = new ProgressReport();
args.Progress?.Report(progressReport);
using var tempMemory = MemoryPool<byte>.Shared.Rent((int)partSize);
for (partNumber = 1; partNumber <= partCount; partNumber++)
{
var dataToCopy = await ReadFullAsync(args.ObjectStreamData, (int)partSize).ConfigureAwait(false);
var bytesRead = await ReadFullAsync(args.ObjectStreamData, tempMemory.Memory).ConfigureAwait(false);
var dataToCopy = tempMemory.Memory[..bytesRead];
if (dataToCopy.IsEmpty && numPartsUploaded > 0) break;
if (partNumber == partCount) expectedReadSize = lastPartSize;
var putObjectArgs = new PutObjectArgs(args)
Expand Down Expand Up @@ -1140,23 +1145,10 @@ await this.ExecuteTaskAsync(requestMessageBuilder,
/// Advances in the stream upto currentPartSize or End of Stream
/// </summary>
/// <param name="data"></param>
/// <param name="currentPartSize"></param>
/// <returns>bytes read in a byte array</returns>
internal static async Task<ReadOnlyMemory<byte>> ReadFullAsync(Stream data, int currentPartSize)
/// <param name="destination"></param>
/// <returns>bytes count read</returns>
internal static async Task<int> ReadFullAsync(Stream data, Memory<byte> destination)
{
Memory<byte> result = new byte[currentPartSize];
var totalRead = 0;
while (totalRead < currentPartSize)
{
var curData = result[totalRead..currentPartSize];
var curRead = await data.ReadAsync(curData).ConfigureAwait(false);
if (curRead == 0) break;
totalRead += curRead;
}

if (totalRead == 0) return null;

// Return only the valid portion without allocating a new buffer
return result[..totalRead];
return await data.ReadAsync(destination).ConfigureAwait(false);
}
}
11 changes: 7 additions & 4 deletions Minio/Helper/S3utils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,16 @@ internal static class S3utils
{
internal static readonly Regex TrimWhitespaceRegex = new("\\s+", RegexOptions.None, TimeSpan.FromHours(1));

internal static readonly Regex AmazonEndpointRegex = new(
"^s3[.-]?(.*?)\\.amazonaws\\.com$",
RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant,
TimeSpan.FromHours(1)
);

internal static bool IsAmazonEndPoint(string endpoint)
{
if (IsAmazonChinaEndPoint(endpoint)) return true;
var rgx = new Regex("^s3[.-]?(.*?)\\.amazonaws\\.com$", RegexOptions.IgnoreCase | RegexOptions.ExplicitCapture,
TimeSpan.FromHours(1));
var matches = rgx.Matches(endpoint);
return matches.Count > 0;
return AmazonEndpointRegex.IsMatch(endpoint);
}

// IsAmazonChinaEndpoint - Match if it is exactly Amazon S3 China endpoint.
Expand Down