Reduce MySqlBulkCopy packet size to 1 MiB.

Author: bgrainger

src/MySqlConnector/MySql.Data.MySqlClient/MySqlBulkCopy.cs

@@ -262,8 +262,7 @@ static void AddColumnMapping(List<MySqlBulkCopyColumnMapping> columnMappings, bo
 
 		internal async Task SendDataReaderAsync(IOBehavior ioBehavior, CancellationToken cancellationToken)
 		{
-			// rent a buffer that can fit in one packet
-			const int maxLength = 16_777_200;
+			const int maxLength = 1048575;
 			var buffer = ArrayPool<byte>.Shared.Rent(maxLength + 1);
 			var outputIndex = 0;
 
@@ -307,7 +306,7 @@ await m_valuesEnumerator.MoveNextAsync().ConfigureAwait(false) :
 					if (!wroteRow)
 					{
 						if (startOutputIndex == 0)
-							throw new NotSupportedException("Total row length must be less than 16MiB.");
+							throw new NotSupportedException("Total row length must be less than 1 MiB.");
 						var payload = new PayloadData(new ArraySegment<byte>(buffer, 0, startOutputIndex));
 						await m_connection.Session.SendReplyAsync(payload, ioBehavior, cancellationToken).ConfigureAwait(false);
 						outputIndex = 0;

tests/SideBySide/BulkLoaderAsync.cs

@@ -396,7 +396,7 @@ public async Task BulkLoadLocalMemoryStream()
 		}
 
 		[Fact]
-		public async Task BulkLoadDataReader()
+		public async Task BulkCopyDataReader()
 		{
 			using var connection = new MySqlConnection(GetLocalConnectionString());
 			using var connection2 = new MySqlConnection(GetLocalConnectionString());
@@ -442,7 +442,7 @@ public void BulkCopyNullDataTable()
 		}
 
 		[Fact]
-		public async Task BulkLoadDataTableWithLongData()
+		public async Task BulkCopyDataTableWithLongData()
 		{
 			var dataTable = new DataTable()
 			{
@@ -453,8 +453,8 @@ public async Task BulkLoadDataTableWithLongData()
 				},
 				Rows =
 				{
-					new object[] { 1, new byte[8388500] },
-					new object[] { 12345678, new byte[8388500] },
+					new object[] { 1, new byte[524200] },
+					new object[] { 12345678, new byte[524200] },
 				},
 			};
 
@@ -474,7 +474,7 @@ public async Task BulkLoadDataTableWithLongData()
 		}
 
 		[Fact]
-		public async Task BulkLoadDataTableWithTooLongData()
+		public async Task BulkCopyDataTableWithTooLongData()
 		{
 			var dataTable = new DataTable()
 			{
@@ -484,7 +484,7 @@ public async Task BulkLoadDataTableWithTooLongData()
 				},
 				Rows =
 				{
-					new object[] { new byte[8388700] },
+					new object[] { new byte[524300] },
 				}
 			};
 
@@ -590,7 +590,7 @@ public async Task BulkCopyAbort(int notifyAfter, int rowCount, int abortAfter, i
 			{
 				Columns = { new DataColumn("value", typeof(string)) },
 			};
-			var str = new string('a', 1_000_000);
+			var str = new string('a', 62500);
 			foreach (var x in Enumerable.Range(1, rowCount))
 				dataTable.Rows.Add(new object[] { str });
 

tests/SideBySide/BulkLoaderSync.cs

@@ -477,7 +477,7 @@ public void BulkLoadLocalMemoryStream()
 		}
 
 		[Fact]
-		public void BulkLoadDataReader()
+		public void BulkCopyDataReader()
 		{
 			using var connection = new MySqlConnection(GetLocalConnectionString());
 			using var connection2 = new MySqlConnection(GetLocalConnectionString());
@@ -522,8 +522,8 @@ public void BulkCopyNullDataTable()
 			Assert.Throws<ArgumentNullException>(() => bulkCopy.WriteToServer(default(DataTable)));
 		}
 
-		[SkippableFact(ServerFeatures.LargePackets)]
-		public void BulkLoadDataTableWithLongBlob()
+		[Fact]
+		public void BulkCopyDataTableWithLongBlob()
 		{
 			var dataTable = new DataTable()
 			{
@@ -534,8 +534,8 @@ public void BulkLoadDataTableWithLongBlob()
 				},
 				Rows =
 				{
-					new object[] { 1, new byte[8388500] },
-					new object[] { 12345678, new byte[8388500] },
+					new object[] { 1, new byte[524200] },
+					new object[] { 12345678, new byte[524200] },
 				},
 			};
 
@@ -555,12 +555,12 @@ public void BulkLoadDataTableWithLongBlob()
 
 			using (var cmd = new MySqlCommand(@"select sum(length(b)) from bulk_load_data_table;", connection))
 			{
-				Assert.Equal(16_777_000m, cmd.ExecuteScalar());
+				Assert.Equal(1_048_400m, cmd.ExecuteScalar());
 			}
 		}
 
-		[SkippableFact(ServerFeatures.LargePackets)]
-		public void BulkLoadDataTableWithLongString()
+		[Fact]
+		public void BulkCopyDataTableWithLongString()
 		{
 			var dataTable = new DataTable()
 			{
@@ -571,8 +571,8 @@ public void BulkLoadDataTableWithLongString()
 				},
 				Rows =
 				{
-					new object[] { 1, new string('a', 16_777_000) },
-					new object[] { 2, new string('b', 16_777_000) },
+					new object[] { 1, new string('a', 1_048_500) },
+					new object[] { 2, new string('b', 1_048_500) },
 				},
 			};
 
@@ -592,12 +592,12 @@ public void BulkLoadDataTableWithLongString()
 
 			using (var cmd = new MySqlCommand(@"select sum(length(b)) from bulk_load_data_table;", connection))
 			{
-				Assert.Equal(33_554_000m, cmd.ExecuteScalar());
+				Assert.Equal(2_097_000m, cmd.ExecuteScalar());
 			}
 		}
 
 		[Fact]
-		public void BulkLoadDataTableWithSpecialCharacters()
+		public void BulkCopyDataTableWithSpecialCharacters()
 		{
 			var dataTable = new DataTable()
 			{
@@ -640,7 +640,7 @@ public void BulkLoadDataTableWithSpecialCharacters()
 		}
 
 		[Fact]
-		public void BulkLoadDataTableWithTooLongBlob()
+		public void BulkCopyDataTableWithTooLongBlob()
 		{
 			var dataTable = new DataTable()
 			{
@@ -650,7 +650,7 @@ public void BulkLoadDataTableWithTooLongBlob()
 				},
 				Rows =
 				{
-					new object[] { new byte[8388700] },
+					new object[] { new byte[524300] },
 				}
 			};
 
@@ -670,7 +670,7 @@ public void BulkLoadDataTableWithTooLongBlob()
 		}
 
 		[Fact]
-		public void BulkLoadDataTableWithTooLongString()
+		public void BulkCopyDataTableWithTooLongString()
 		{
 			var dataTable = new DataTable()
 			{
@@ -680,7 +680,7 @@ public void BulkLoadDataTableWithTooLongString()
 				},
 				Rows =
 				{
-					new object[] { new string('a', 16_777_400) },
+					new object[] { new string('a', 1_048_700) },
 				}
 			};
 
@@ -775,7 +775,7 @@ public void BulkCopyAbort(int notifyAfter, int rowCount, int abortAfter, int exp
 			{
 				Columns = { new DataColumn("value", typeof(string)) },
 			};
-			var str = new string('a', 1_000_000);
+			var str = new string('a', 62500);
 			foreach (var x in Enumerable.Range(1, rowCount))
 				dataTable.Rows.Add(new object[] { str });