Fix a race condition in TCP epoll code (#4993)

t's a race between PerfClientWorker::WorkerThread and TcpWorker::WorkerThread. In the middle of creating a socket from the perf client worker thread, as long as epoll has been told to monitor the socket, a notification could come up at the same time from tcp worker thread and frees the connection.

#1  0x00005b7e577eec9f in CxPlatSocketContextSetEvents
#2  0x00005b7e577ef725 in CxPlatSocketCreateTcpInternal
#3  0x00005b7e577efb8a in SocketCreateTcp
#4  0x00005b7e577ebdb9 in CxPlatSocketCreateTcp
#5  0x00005b7e577db274 in TcpConnection::Start
#6  0x00005b7e577def35 in PerfClientConnection::Initialize
#7  0x00005b7e577df124 in PerfClientWorker::StartNewConnection
#8  0x00005b7e577df140 in PerfClientWorker::WorkerThread

The fix is moving the actual connection start to the tcp worker thread.

Author: csujedihy

Diff for: src/perf/lib/Tcp.cpp

@@ -404,6 +404,7 @@ TcpConnection::TcpConnection(
         WriteOutput("SecConfig load FAILED\n");
         return;
     }
+    QuicAddrSetFamily(&Route.LocalAddress, QUIC_ADDRESS_FAMILY_UNSPEC);
     Initialized = true;
 }
 
@@ -422,7 +423,9 @@ TcpConnection::Start(
     }
     if (LocalAddress) {
         Family = QuicAddrGetFamily(LocalAddress);
+        Route.LocalAddress = *LocalAddress;
     }
+
     if (RemoteAddress) {
         Route.RemoteAddress = *RemoteAddress;
     } else {
@@ -437,15 +440,7 @@ TcpConnection::Start(
         }
     }
     QuicAddrSetPort(&Route.RemoteAddress, ServerPort);
-    if (QUIC_FAILED(
-        CxPlatSocketCreateTcp(
-            Datapath,
-            LocalAddress,
-            &Route.RemoteAddress,
-            this,
-            &Socket))) {
-        return false;
-    }
+    ConnStartQueued = true;
     Queue();
     return true;
 }
@@ -627,6 +622,19 @@ TcpConnection::TlsReceiveTicketCallback(
 
 void TcpConnection::Process()
 {
+    if (ConnStartQueued) {
+        ConnStartQueued = false;
+        if (QUIC_FAILED(
+            CxPlatSocketCreateTcp(
+                Datapath,
+                QuicAddrGetFamily(&Route.LocalAddress) == QUIC_ADDRESS_FAMILY_UNSPEC ?
+                    nullptr : &Route.LocalAddress,
+                &Route.RemoteAddress,
+                this,
+                &Socket))) {
+            Shutdown = true;
+        }
+    }
     if (IndicateAccept) {
         IndicateAccept = false;
         TcpServer* Server = (TcpServer*)Context;

Diff for: src/perf/lib/Tcp.h

@@ -183,6 +183,7 @@ class TcpConnection {
     bool Closed{false};
     bool QueuedOnWorker{false};
     bool StartTls{false};
+    bool ConnStartQueued{false};
     bool IndicateAccept{false};
     bool IndicateConnect{false};
     bool IndicateSendComplete{false};