Feature/s3 proxy final

.gitignore

@@ -123,3 +123,4 @@ BaseSpace.SDK/src/BaseSpace.SDK.MonoTouch/BaseSpace.SDK.MonoTouch.Tests/Settings
 *.plist
 *.DS_Store
 *.DS_Store
+.vs/config/applicationhost.config

BaseSpace.SDK.Tests/BaseSpace.SDK.Tests.csproj

@@ -46,17 +46,17 @@
     <Reference Include="log4net">
       <HintPath>$(SolutionDir)\packages\log4net.1.2.10\lib\2.0\log4net.dll</HintPath>
     </Reference>
-    <Reference Include="ServiceStack.Common, Version=3.9.70.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="ServiceStack.Common, Version=3.9.35.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>$(SolutionDir)\packages\ServiceStack.Common.3.9.35\lib\net35\ServiceStack.Common.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Common.dll</HintPath>
     </Reference>
-    <Reference Include="ServiceStack.Interfaces, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="ServiceStack.Interfaces, Version=3.9.35.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>$(SolutionDir)\packages\ServiceStack.Common.3.9.35\lib\net35\ServiceStack.Interfaces.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Interfaces.dll</HintPath>
     </Reference>
-    <Reference Include="ServiceStack.Text, Version=3.9.70.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="ServiceStack.Text, Version=3.9.35.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>$(SolutionDir)\packages\ServiceStack.Text.3.9.35\lib\net35\ServiceStack.Text.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Text.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />

BaseSpace.SDK.Tests/Integration/BaseIntegrationTest.cs

@@ -25,6 +25,11 @@ public BaseIntegrationTest()
             Debug.Listeners.Add(new DefaultTraceListener());
         }
 
+        private IBaseSpaceClient CreateRealClient()
+        {
+            return CreateRealClient(false);
+        }
+
         private readonly Lazy<IBaseSpaceClient> _lazy;
 
         public IBaseSpaceClient Client
@@ -43,7 +48,7 @@ protected ILog Log
         }
 
         // Note: prefer access through the Client property!
-        protected virtual IBaseSpaceClient CreateRealClient()
+        protected virtual IBaseSpaceClient CreateRealClient(bool useS3Proxy= false)
         {
             //string apiKey = ConfigurationManager.AppSettings.Get("basespace:api-key");
             //string apiSecret = ConfigurationManager.AppSettings.Get("basespace:api-secret");
@@ -56,8 +61,9 @@ protected virtual IBaseSpaceClient CreateRealClient()
 					Authentication = GetAuthentication(),
 					BaseSpaceApiUrl = apiUrl, 
 					BaseSpaceWebsiteUrl = webUrl, 
-					Version = version
-				};
+					Version = version,
+                    UseS3Proxy = useS3Proxy
+            };
 
 			return new BaseSpaceClient(settings);
         }

BaseSpace.SDK.Tests/Integration/FileDownloadTests.cs

@@ -13,6 +13,33 @@ namespace Illumina.BaseSpace.SDK.Tests.Integration
 {
     public class FileDownloadTests : BaseIntegrationTest
     {
+        [Fact]
+        public void CanDownloadSmallFile()
+        {
+            IBaseSpaceClient client = CreateRealClient(useS3Proxy:false);
+            var fs = new FileStream("DownloadedFile-" + StringHelpers.RandomAlphanumericString(5), FileMode.OpenOrCreate);           
+            client.FileDownloadProgressChanged += delegate(object sender, FileDownloadProgressChangedEventArgs e)
+            {                          
+               Console.WriteLine($"FileId={e.FileId} BytesDownloaded={e.BytesDownloaded} Progress={e.ProgressPercentage}%");            
+            };
+
+            client.DownloadFile("145435603", fs);            
+            Assert.Equal(fs.Length, 64596);            
+        }
+
+        [Fact]
+        public void CanDownloadSmallFileWithProxy()
+        {
+            IBaseSpaceClient client = CreateRealClient(useS3Proxy: true);
+            var fs = new FileStream("DownloadedFile-" + StringHelpers.RandomAlphanumericString(5), FileMode.OpenOrCreate);
+
+            client.FileDownloadProgressChanged += delegate (object sender, FileDownloadProgressChangedEventArgs e)
+            {
+                Debug.WriteLine($"FileId={e.FileId} BytesDownloaded={e.BytesDownloaded} Progress={e.ProgressPercentage}%");
+            };
+            client.DownloadFile("145435603", fs);
+            Assert.Equal(fs.Length, 64596);
+        }
         [Fact]
         public void CanDownloadFile()
         {
@@ -35,8 +62,7 @@ public void CanDownloadFile()
             Client.DownloadFile(response.Response.Id, fs);
             Assert.Equal(fs.Length, data.Length);
         }
-
-        [Fact]
+        [Fact (Skip="takes too long - sujit")]
         public void CanDownloadLargerFile()
         {
             var project = TestHelpers.CreateRandomTestProject(Client);

BaseSpace.SDK.sln

@@ -16,6 +16,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{E6DCBA
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TerminalVelocity.Sharp", "..\TerminalVelocity\src\TerminalVelocity.Sharp\TerminalVelocity.Sharp.csproj", "{9FE91B4B-5C87-4AC8-B15D-449987CAF4DA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Starksoft.Aspen", "..\..\Users\Sujit\Downloads\starksoft-aspen-master\Starksoft.Aspen\Starksoft.Aspen.csproj", "{5F228FCD-54F1-491D-93CE-3D768655FADC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -48,6 +50,14 @@ Global
 		{9FE91B4B-5C87-4AC8-B15D-449987CAF4DA}.XamarinMacDebug|Any CPU.Build.0 = XamarinMacDebug|Any CPU
 		{9FE91B4B-5C87-4AC8-B15D-449987CAF4DA}.XamarinMacRelease|Any CPU.ActiveCfg = XamarinMacRelease|Any CPU
 		{9FE91B4B-5C87-4AC8-B15D-449987CAF4DA}.XamarinMacRelease|Any CPU.Build.0 = XamarinMacRelease|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.XamarinMacDebug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.XamarinMacDebug|Any CPU.Build.0 = Debug|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.XamarinMacRelease|Any CPU.ActiveCfg = Release|Any CPU
+		{5F228FCD-54F1-491D-93CE-3D768655FADC}.XamarinMacRelease|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

BaseSpace.SDK/BaseSpace.SDK.csproj

@@ -48,13 +48,13 @@
       <HintPath>..\packages\Common.Logging.2.1.2\lib\net40\Common.Logging.dll</HintPath>
     </Reference>
     <Reference Include="ServiceStack.Common">
-      <HintPath>..\..\..\SignedBinaries\ServiceStack.Common.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Common.dll</HintPath>
     </Reference>
     <Reference Include="ServiceStack.Interfaces">
-      <HintPath>..\..\..\SignedBinaries\ServiceStack.Interfaces.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Interfaces.dll</HintPath>
     </Reference>
     <Reference Include="ServiceStack.Text">
-      <HintPath>..\..\..\SignedBinaries\ServiceStack.Text.dll</HintPath>
+      <HintPath>..\..\SpaceElevator\SignedBinaries\ServiceStack.Text.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />

BaseSpace.SDK/Infrastructure/BaseSpaceClientSettings.cs

@@ -44,5 +44,7 @@ public BaseSpaceClientSettings()
         public IAuthentication Authentication { get; set; }
 
         public int TimeoutMin { get; set; }
-	}
+
+        public bool UseS3Proxy { get; set; }
+    }
 }

BaseSpace.SDK/Infrastructure/DownloadFileCommand.cs

@@ -28,9 +28,15 @@ internal class DownloadFileCommand : IAsyncProgress<LargeFileDownloadProgressCha
         public DownloadFileCommand(BaseSpaceClient client, string fileId, Stream stream, IClientSettings settings, CancellationToken token = new CancellationToken(), IWebProxy proxy = null, bool enableLogging = true)
         {
             DateTime expiration;
-            string url = GetFileContentUrl(client,fileId, out expiration);
+            long fileSize;
+
+            string url = GetFileContentUrl(client, fileId, out expiration, out fileSize, settings.UseS3Proxy);
 #pragma warning disable 618
-            ILargeFileDownloadParameters parameters = new LargeFileDownloadWithStreamParameters(new Uri(url), stream, 0, id: fileId, maxThreads: DEFAULT_THREADS, maxChunkSize: (int)settings.FileDownloadMultipartSizeThreshold, autoCloseStream: false, verifyLength: true);
+            ILargeFileDownloadParameters parameters = new LargeFileDownloadWithStreamParameters
+                (
+                 new Uri(url), stream, fileSize, id: fileId, maxThreads: DEFAULT_THREADS, maxChunkSize: (int)settings.FileDownloadMultipartSizeThreshold, 
+                 autoCloseStream: false, verifyLength: !settings.UseS3Proxy, accessToken: settings.UseS3Proxy ? settings.Authentication.AccessToken : null
+                );
 #pragma warning restore 618
             _parameters = parameters;
             _token = token;
@@ -43,8 +49,14 @@ public DownloadFileCommand(BaseSpaceClient client, V1pre3FileCompact file, strin
                                    IClientSettings settings, CancellationToken token = new CancellationToken(), bool enableLogging = true, int threadCount = DEFAULT_THREADS)
         {
             DateTime expiration;
-            string url = GetFileContentUrl(client, file.Id, out expiration);
-            ILargeFileDownloadParameters parameters = new LargeFileDownloadParameters(new Uri(url), targetFileName, maxThreads: threadCount, maxChunkSize: (int?)settings.FileDownloadMultipartSizeThreshold, id: file.Id);
+            long fileSize;
+            string url = GetFileContentUrl(client, file.Id, out expiration, out fileSize, settings.UseS3Proxy);
+
+            ILargeFileDownloadParameters parameters = new LargeFileDownloadParameters
+                (
+                    new Uri(url), targetFileName, fileSize: fileSize, maxThreads: threadCount, maxChunkSize: (int?)settings.FileDownloadMultipartSizeThreshold, 
+                    id: file.Id, verifyLength: !settings.UseS3Proxy, accessToken: settings.UseS3Proxy?settings.Authentication.AccessToken:null
+                );
             _parameters = parameters;
             _token = token;
             _enableLogging = enableLogging;
@@ -53,9 +65,14 @@ public DownloadFileCommand(BaseSpaceClient client, V1pre3FileCompact file, strin
         public DownloadFileCommand(BaseSpaceClient client, V1pre3FileCompact file, string targetFileName, IClientSettings settings, int threadCount,  int maxChunkSize, CancellationToken token = new CancellationToken(), bool enableLogging = true)
         {
             DateTime expiration;
-            string url = GetFileContentUrl(client, file.Id, out expiration);
+            long fileSize;
+            string url = GetFileContentUrl(client, file.Id, out expiration, out fileSize, settings.UseS3Proxy);
             _fileName = string.Format("[{0}],{1}",file.Id,file.Name);
-            ILargeFileDownloadParameters parameters = new LargeFileDownloadParameters(new Uri(url), targetFileName, maxThreads: threadCount, maxChunkSize: maxChunkSize, id: file.Id);
+            ILargeFileDownloadParameters parameters = new LargeFileDownloadParameters
+                (
+                    new Uri(url), targetFileName, fileSize: fileSize, maxThreads: threadCount, maxChunkSize: maxChunkSize, id: file.Id, 
+                    verifyLength: !settings.UseS3Proxy, accessToken: settings.UseS3Proxy ? settings.Authentication.AccessToken : null
+                );
             _parameters = parameters;
             _token = token;
             _enableLogging = enableLogging;
@@ -117,31 +134,47 @@ public void Report(LargeFileDownloadProgressChangedEventArgs e)
 		}
 
 	    protected virtual void OnFileDownloadProgressChanged(FileDownloadProgressChangedEventArgs e)
-		{
-			if (FileDownloadProgressChanged != null)
-			{
-				FileDownloadProgressChanged(this, e);
-			}
-		}
-
-		private static string GetFileContentUrl(BaseSpaceClient client, string fileId, out DateTime expiration)
-		{
-			// get the download URL
-			var response = client.GetFileContentUrl(new FileContentRedirectMetaRequest(fileId));
-
-			if (response.Response == null || response.Response.HrefContent == null)
-			{
-				throw new ApplicationException("Unable to get HrefContent");
-			}
-
-			if (!response.Response.SupportsRange)
-			{
-				throw new ApplicationException("This file does not support range queries");
-			}
-
-			expiration = response.Response.Expires;
-			return response.Response.HrefContent;
-		}
+        {
+            FileDownloadProgressChanged?.Invoke(this, e);
+        }
+        private static string GetFileContentUrl(BaseSpaceClient client, string fileId, out DateTime expiration, out long fileSize, bool useS3Redirect)
+        {
+            if (useS3Redirect)
+            {
+                var response = client.GetFilesInformation(new GetFileInformationRequest(fileId));
+                if (response.Response?.HrefContent == null)
+                {
+                    throw new ApplicationException("Unable to get HrefContent");
+                }
+                expiration = DateTime.MaxValue;
+
+                var apiUri = new Uri(client.Settings.BaseSpaceApiUrl);
+                var uriBuilder = new System.UriBuilder(apiUri);
+                uriBuilder.Path += response.Response.HrefContent;
+                uriBuilder.Query = "redirect=proxy";            
+                fileSize = response.Response.Size;
+                return uriBuilder.Uri.ToString();
+            }
+            else
+            {
+                // get the download URL
+                var response = client.GetFileContentUrl(new FileContentRedirectMetaRequest(fileId));
+
+                if (response.Response?.HrefContent == null)
+                {
+                    throw new ApplicationException("Unable to get HrefContent");
+                }
+
+                if (!response.Response.SupportsRange)
+                {
+                    throw new ApplicationException("This file does not support range queries");
+                }
+
+                expiration = response.Response.Expires;
+                fileSize = -1;
+                return response.Response.HrefContent;
+            }
+        }
 
 	}
 }

BaseSpace.SDK/Infrastructure/OAuth2Authentication.cs

@@ -4,12 +4,10 @@
 namespace Illumina.BaseSpace.SDK
 {
 	public class OAuth2Authentication : IAuthentication
-	{
-		private readonly string accessToken;
-
+	{		
 		public OAuth2Authentication(string accessToken)
 		{
-			this.accessToken = accessToken;
+            AccessToken = accessToken;
 		}
 
 		public OAuth2Authentication(string appId, string appSecret)
@@ -22,16 +20,18 @@ public OAuth2Authentication(string appId, string appSecret)
 
 		public string AppSecret { get; private set; }
 
-		public void UpdateHttpHeader(HttpWebRequest request)
+        public string AccessToken { get; private set; }
+
+        public void UpdateHttpHeader(HttpWebRequest request)
 		{
 			UpdateHttpHeader(request.Headers, request.RequestUri, request.Method);
 		}
 
         public void UpdateHttpHeader(WebHeaderCollection headers, Uri requestUri, string requestMethod)
 		{
-			if (headers != null && !string.IsNullOrEmpty(accessToken))
+			if (headers != null && !string.IsNullOrEmpty(AccessToken))
 			{
-				headers.Add("Authorization", string.Format("Bearer {0}", accessToken));
+				headers.Add("Authorization", string.Format("Bearer {0}", AccessToken));
 			}
 		}
 	}

BaseSpace.SDK/Interfaces/IAuthentication.cs

@@ -7,5 +7,6 @@ public interface IAuthentication
 	{
 		void UpdateHttpHeader(HttpWebRequest request);
 	    void UpdateHttpHeader(WebHeaderCollection headers, Uri requestUri, string requestMethod);
-	}
+        string AccessToken { get; }
+    }
 }

BaseSpace.SDK/Interfaces/IClientSettings.cs

@@ -18,5 +18,6 @@ public interface IClientSettings
 		IAuthentication Authentication { get; }
 
         int TimeoutMin { get;  }
-	}
+        bool UseS3Proxy { get; }
+    }
 }