Fix memory efficiency problem with statistics loader

Author: Jroland

src/StatisticsTestLoader/DestinationKafka.cs

@@ -20,7 +20,7 @@ public DestinationKafka(params Uri[] servers)
         {
             var options = new KafkaOptions(servers) { Log = new ConsoleLogger() };
             _router = new BrokerRouter(options);
-            _producer = new Producer(_router, maximumMessageBuffer: 5000, maximumAsyncRequests: 10) { BatchSize = 1000, BatchDelayTime = TimeSpan.FromSeconds(5) };
+            _producer = new Producer(_router, maximumMessageBuffer: 5000, maximumAsyncRequests: 10) { BatchSize = 1000, BatchDelayTime = TimeSpan.FromSeconds(1) };
 
             StatisticsTracker.OnStatisticsHeartbeat += StatisticsTracker_OnStatisticsHeartbeat;
         }
@@ -79,7 +79,7 @@ public async Task PostBatchAsync(List<KafkaRecord> records)
                 {
                     Console.WriteLine("POSTING: {0}, {1}", topicBatch.Count(), _producer.BufferCount);
                     var result = await _producer
-                        .SendMessageAsync(topicBatch.Key, topicBatch.Select(x => new Message(x.Record, x.Key)), acks: 1, codec: MessageCodec.CodecGzip)
+                        .SendMessageAsync(topicBatch.Key, topicBatch.Select(x => x.Message), acks: 0, codec: MessageCodec.CodecGzip)
                         .ConfigureAwait(false);
 
                     if (result.Any(x => x.Error != (int)ErrorResponseCode.NoError))
@@ -88,9 +88,7 @@ public async Task PostBatchAsync(List<KafkaRecord> records)
                     }
                     else
                     {
-                        var offset = topicBatch.Max(x => x.Offset);
                         await SetStoredOffsetAsync(topicBatch.Key, topicBatch.Max(x => x.Offset)).ConfigureAwait(false);
-                        //Console.WriteLine("Topic:{0} Count:{1} Offset:{2}", topicBatch.Key, topicBatch.Count(), offset);
                     }
                 }
             }

src/StatisticsTestLoader/KafkaRecord.cs

@@ -1,10 +1,17 @@
-namespace StatisticsTestLoader
+using KafkaNet.Protocol;
+
+namespace StatisticsTestLoader
 {
     public class KafkaRecord
     {
         public string Key { get; set; }
         public string Topic { get; set; }
         public long Offset { get; set; }
-        public string Record { get; set; }
+        public Message Message { get; private set; }
+
+        public void AddDocument(string document)
+        {
+            Message = new Message(Key, document);
+        }
     }
 }

src/StatisticsTestLoader/Program.cs

@@ -42,7 +42,7 @@ private static void StartPolling(IRecordSource source, DestinationKafka kafka)
                 try
                 {
                     var index = kafka.GetStoredOffset(source.Topic);
-                    foreach (var batchEnumerable in source.Poll(index).Batch(1000))
+                    foreach (var batchEnumerable in source.Poll(index).Batch(500))
                     {
                         var batch = batchEnumerable.ToList();
 

src/StatisticsTestLoader/SourcePropertyChanges.cs

@@ -77,7 +77,7 @@ private void PopulateData(long index, BlockingCollection<KafkaRecord> data)
                         {
                             if (records.ContainsKey(change.Key))
                             {
-                                change.Record = records[change.Key].ToString();
+                                change.AddDocument(records[change.Key].ToString());
                                 data.Add(change);
                             }
                         }