proof of concept for message durability with small files (fit a batch request)

.gitignore

@@ -1,4 +1,5 @@
 # Created by https://www.gitignore.io
+analytics/pending
 
 ### Maven ###
 target/
@@ -128,4 +129,4 @@ atlassian-ide-plugin.xml
 .classpath
 .project
 .settings/
-.factorypath
+.factorypath

ARQUITECTURE.md

@@ -0,0 +1,60 @@
+```mermaid
+
+sequenceDiagram    
+    box Data Consolidation
+        participant ConsolidationService
+    end
+
+        participant SegmentClient
+    box HTTP
+        participant QueueHttp        
+        participant LooperHttp        
+        participant SegmentAPI
+    end
+    box File
+        participant QueueFile
+        participant WriteFile
+        participant File
+        participant WatchFile
+    end
+
+    activate ConsolidationService
+    ConsolidationService->>+SegmentClient: enqueue    
+    SegmentClient<<->>QueueHttp: offer 
+    alt QueueHttp overflow
+        SegmentClient<<->>QueueFile: put
+    end
+    SegmentClient->>-ConsolidationService: 
+    deactivate ConsolidationService
+
+    loop consume QueueHttp
+        LooperHttp->>QueueHttp:take
+        activate LooperHttp       
+    end
+    LooperHttp->>SegmentAPI: batchUpload
+    note over LooperHttp,SegmentAPI: Batch
+    note over LooperHttp,SegmentAPI: CircuitBreaker and Retry
+    note over LooperHttp,SegmentAPI: HTTP requests submited to a pool
+    deactivate LooperHttp
+
+    alt retry exhausted or circuit open
+        note over LooperHttp: pool threads
+        LooperHttp->>QueueFile: put
+    end
+
+    loop consume QueueFile
+        WriteFile->>QueueFile:take
+        activate WriteFile       
+    end
+    WriteFile->>File: write
+    note over WriteFile: Batch and save file
+    deactivate WriteFile
+
+    note over WatchFile: check last written 
+    activate WatchFile
+    loop watch QueueFile        
+        WatchFile->>File: read and remove
+        WatchFile->>QueueHttp: offer
+    end
+    deactivate WatchFile
+```

analytics-core/src/main/java/com/segment/analytics/http/SegmentService.java

@@ -2,6 +2,7 @@
 
 import com.segment.analytics.messages.Batch;
 import okhttp3.HttpUrl;
+import okhttp3.RequestBody;
 import retrofit2.Call;
 import retrofit2.http.Body;
 import retrofit2.http.POST;
@@ -11,4 +12,7 @@
 public interface SegmentService {
   @POST
   Call<UploadResponse> upload(@Url HttpUrl uploadUrl, @Body Batch batch);
+
+  @POST
+  Call<UploadResponse> upload(@Url HttpUrl uploadUrl, @Body RequestBody batch);
 }

analytics/pom.xml

@@ -39,9 +39,21 @@
       <artifactId>findbugs</artifactId>
       <scope>provided</scope>
     </dependency>
+
     <dependency>
-      <groupId>com.segment.backo</groupId>
-      <artifactId>backo</artifactId>
+      <groupId>dev.failsafe</groupId>
+      <artifactId>failsafe</artifactId>
+      <version>3.3.2</version>
+    </dependency>
+    <dependency>
+      <groupId>dev.failsafe</groupId>
+      <artifactId>failsafe-retrofit</artifactId>
+      <version>3.3.2</version>
+   </dependency>
+   <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.18.0</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -67,7 +79,19 @@
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
-    </dependency>
+    </dependency>    
+	<dependency>
+      <groupId>org.wiremock</groupId>
+	  <artifactId>wiremock-standalone</artifactId>
+	  <version>3.2.0</version>
+      <scope>test</scope>
+	</dependency>
+	<dependency>
+      <groupId>org.awaitility</groupId>
+      <artifactId>awaitility</artifactId>
+      <version>4.2.2</version>
+      <scope>test</scope>
+    </dependency>    
   </dependencies>
 
   <build>
@@ -84,6 +108,18 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>3.5.3</version> 
+        <configuration>
+          <systemPropertyVariables>
+            <java.util.logging.config.file>
+              ${project.basedir}/src/test/resources/logging.properties
+            </java.util.logging.config.file>
+          </systemPropertyVariables>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>