fix: feature DETECT_STDOUT_ENCODING fixes #1 fixes #2

Author: labbbirder

Editor/ConsoleUtils.cs

@@ -87,16 +87,16 @@ public static string ScanColorLog(string input, ColorMarkVisitor visitor)
         /// </summary>
         /// <param name="bytes"></param>
         /// <returns></returns>
-        public unsafe static bool IsUTF8InsteadOf16(byte[] bytes)
+        public unsafe static bool IsValidUTF8(byte[] bytes, int index, int count)
         {
-            if (bytes.Length % 2 != 0) return true; // distinguish from utf-16 only
-            if (!bytes.Any(b => b == 0))
-            {
-                _ = 0;
-            }
+            // if (bytes.Length % 2 != 0) return true; // distinguish from utf-16 only
             var c = 0;
-            foreach (var b in bytes)
+            for (int i = index; i < index + count; i++)
             {
+                var b = bytes[i];
+                if (b == 0)
+                    return false;
+
                 if (c != 0)
                 {
                     c--;

Editor/Shell.cs

@@ -9,6 +9,8 @@
 using System.Threading;
 using System.Collections.Concurrent;
 using System.Runtime.InteropServices;
+using System.Buffers;
+using System.Linq;
 
 namespace com.bbbirder.unityeditor
 {
@@ -55,6 +57,7 @@ public struct ShellSettings
 
 	public static class Shell
 	{
+		readonly static Encoding DEFAULT_WINDOWS_CONSOLE_ENCODING = Encoding.GetEncoding("gbk");
 		public static Dictionary<string, string> DefaultEnvironment = new();
 		internal static ConcurrentQueue<(ShellRequest req, LogEventType type, object arg)> queue = new();
 		public static Dictionary<ShellRequest, StringBuilder> lineBuilders = new();
@@ -135,14 +138,14 @@ static Process CreateProcess(string cmd, string workDirectory = ".", Dictionary<
 			start.RedirectStandardError =
 			start.RedirectStandardInput = true;
 
-			// 			start.StandardInputEncoding =
-			// 			start.StandardOutputEncoding =
-			// 			start.StandardErrorEncoding =
-			// #if UNITY_EDITOR_WIN && DETECT_STDOUT_ENCODING
-			// 				Encoding.Unicode;
-			// #else
-			// 				Encoding.UTF8;
-			// #endif
+			start.StandardInputEncoding =
+			start.StandardOutputEncoding =
+			start.StandardErrorEncoding =
+#if UNITY_EDITOR_WIN && DETECT_STDOUT_ENCODING
+				Encoding.Unicode;
+#else
+				Encoding.UTF8;
+#endif
 
 			start.ArgumentList.Clear();
 			foreach (var arg in args)
@@ -179,7 +182,7 @@ public static ShellRequest RunCommand(string cmd, ShellSettings? settings = defa
 				"@echo off>nul\n" +
 #endif
 #if UNITY_EDITOR_WIN && DETECT_STDOUT_ENCODING
-				"@chcp 65001>nul\n"
+				"@chcp 65001>nul\n" +
 #endif
 				cmd;
 
@@ -222,6 +225,56 @@ public static ShellRequest RunCommandLine(string executable, params object[] arg
 			return RunCommandLine(executable, ShellSettings.Default, args);
 		}
 
+		static IEnumerable<string> GetConsoleOutput(StreamReader console)
+		{
+			const int BUFFER_SIZE = 40960;
+			var buffer = ArrayPool<byte>.Shared.Rent(BUFFER_SIZE);
+			while (!console.EndOfStream)
+			{
+				var index = 0;
+				var wchar = console.CurrentEncoding == Encoding.Unicode;
+				while (console.Peek() != -1)
+				{
+					var b = console.Read();
+					if (wchar)
+					{
+						if (index + 2 < buffer.Length)
+						{
+							if (BitConverter.IsLittleEndian)
+							{
+								buffer[index++] = (byte)(b & 0xff);
+								buffer[index++] = (byte)(b >> 8);
+							}
+							else
+							{
+								buffer[index++] = (byte)(b >> 8);
+								buffer[index++] = (byte)(b & 0xff);
+							}
+						}
+					}
+					else
+					{
+						if (index + 1 < buffer.Length)
+						{
+							buffer[index++] = (byte)b;
+						}
+					}
+				}
+
+				var encoding = Encoding.UTF8;
+#if UNITY_EDITOR_WIN && DETECT_STDOUT_ENCODING
+				UnityEngine.Debug.Log(string.Join(',', buffer.Select(b => b.ToString("x2")).ToArray(), 0, index));
+				UnityEngine.Debug.Log(ConsoleUtils.IsValidUTF8(buffer, 0, index) + " " + index);
+				if (!ConsoleUtils.IsValidUTF8(buffer, 0, index))
+				{
+					encoding = DEFAULT_WINDOWS_CONSOLE_ENCODING;
+				}
+#endif
+				var output = encoding.GetString(buffer, 0, index);
+				yield return output;
+			}
+			ArrayPool<byte>.Shared.Return(buffer);
+		}
 
 		static ShellRequest QueueUpProcess(Process p, string cmd, ShellSettings settings)
 		{
@@ -232,32 +285,18 @@ static ShellRequest QueueUpProcess(Process p, string cmd, ShellSettings settings
 			{
 				try
 				{
-					var builder = new StringBuilder();
-
-					while (!p.StandardOutput.EndOfStream)
+					foreach (var output in GetConsoleOutput(p.StandardOutput))
 					{
-						while (p.StandardOutput.Peek() > -1)
-						{
-							builder.Append((char)p.StandardOutput.Read());
-						}
-						var output = builder.ToString();
-						builder.Clear();
 						queue.Enqueue((req, LogEventType.InfoLog, output));
 					}
 
-					while (!p.StandardError.EndOfStream)
+					foreach (var output in GetConsoleOutput(p.StandardError))
 					{
-						string error = p.StandardError.ReadLine();
-
-						if (!string.IsNullOrEmpty(error))
-						{
-							if (!string.IsNullOrEmpty(error))
-								queue.Enqueue((req, LogEventType.ErrorLog, error));
-						}
+						if (!string.IsNullOrEmpty(output))
+							queue.Enqueue((req, LogEventType.ErrorLog, output));
 					}
 
 					queue.Enqueue((req, LogEventType.EndStream, p.ExitCode));
-
 				}
 				catch (Exception e)
 				{

Editor/ShellRequest.cs

@@ -149,14 +149,6 @@ void OnLog(LogEventType type, string log)
         {
             result.AppendLine(type, log);
 
-#if UNITY_EDITOR_WIN && DETECT_STDOUT_ENCODING
-            var bytes = Encoding.Unicode.GetBytes(log);
-            if (ConsoleUtils.IsUTF8InsteadOf16(bytes))
-            {
-                log = Encoding.UTF8.GetString(bytes);
-            }
-#endif
-
             onLog?.Invoke(type, log);
 
             if (!settings.quiet)