diff --git a/osu.Framework.Benchmarks/BenchmarkTextBuilder.cs b/osu.Framework.Benchmarks/BenchmarkTextBuilder.cs index 73cd45f0fc..9001e90654 100644 --- a/osu.Framework.Benchmarks/BenchmarkTextBuilder.cs +++ b/osu.Framework.Benchmarks/BenchmarkTextBuilder.cs @@ -35,6 +35,14 @@ public void RemoveLastCharacterWithDifferentBaselines() textBuilder.RemoveLastCharacter(); } + [Benchmark] + public void AddText() + { + textBuilder = new TextBuilder(store, FontUsage.Default); + textBuilder.AddText( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat."); + } + private void initialiseBuilder(bool withDifferentBaselines) { textBuilder = new TextBuilder(store, FontUsage.Default); @@ -42,16 +50,16 @@ private void initialiseBuilder(bool withDifferentBaselines) char different = 'B'; for (int i = 0; i < 100; i++) - textBuilder.AddCharacter(withDifferentBaselines && (i % 10 == 0) ? different++ : 'A'); + textBuilder.AddCharacter(new Grapheme(withDifferentBaselines && (i % 10 == 0) ? different++ : 'A')); } private class TestStore : ITexturedGlyphLookupStore { - public ITexturedCharacterGlyph Get(string? fontName, char character) => new TexturedCharacterGlyph( - new CharacterGlyph(character, character, character, character, character, null), + public ITexturedCharacterGlyph Get(string? fontName, Grapheme character) => new TexturedCharacterGlyph( + new CharacterGlyph(character, character.CharValue, character.CharValue, character.CharValue, character.CharValue, null), new DummyRenderer().CreateTexture(1, 1)); - public Task GetAsync(string fontName, char character) => Task.Run(() => Get(fontName, character)); + public Task GetAsync(string fontName, Grapheme character) => Task.Run(() => Get(fontName, character)); } } } diff --git a/osu.Framework.Templates/templates/template-flappy/FlappyDon.Game/Elements/ScoreSpriteText.cs b/osu.Framework.Templates/templates/template-flappy/FlappyDon.Game/Elements/ScoreSpriteText.cs index 88c383098e..5f6818733c 100644 --- a/osu.Framework.Templates/templates/template-flappy/FlappyDon.Game/Elements/ScoreSpriteText.cs +++ b/osu.Framework.Templates/templates/template-flappy/FlappyDon.Game/Elements/ScoreSpriteText.cs @@ -45,7 +45,7 @@ public ScoreGlyphStore(TextureStore textures) this.textures = textures; } - public ITexturedCharacterGlyph Get(string fontName, char character) + public ITexturedCharacterGlyph Get(string fontName, Grapheme character) { var texture = textures.Get($"{character}"); @@ -56,7 +56,7 @@ public ITexturedCharacterGlyph Get(string fontName, char character) texture.Width, 0, null), texture, 0.09f); } - public Task GetAsync(string fontName, char character) => Task.Run(() => Get(fontName, character)); + public Task GetAsync(string fontName, Grapheme character) => Task.Run(() => Get(fontName, character)); } } } diff --git a/osu.Framework.Tests/Text/GraphemeTest.cs b/osu.Framework.Tests/Text/GraphemeTest.cs new file mode 100644 index 0000000000..de301e81ea --- /dev/null +++ b/osu.Framework.Tests/Text/GraphemeTest.cs @@ -0,0 +1,456 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Linq; +using System.Text; +using NUnit.Framework; +using osu.Framework.Text; + +namespace osu.Framework.Tests.Text +{ + [TestFixture] + public class GraphemeTest + { + #region Constructor Tests + + /// + /// Tests that the char constructor correctly creates a BMP grapheme. + /// + [Test] + public void TestCharConstructor() + { + var grapheme = new Grapheme('a'); + + Assert.That(grapheme.CharValue, Is.EqualTo('a')); + Assert.That(grapheme.StringValue, Is.Null); + Assert.That(grapheme.IsBmp, Is.True); + Assert.That(grapheme.IsSurrogate, Is.False); + Assert.That(grapheme.IsSingleScalarValue, Is.True); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(1)); + } + + /// + /// Tests that the string constructor correctly handles single BMP characters. + /// + [Test] + public void TestStringConstructorWithSingleCharacter() + { + var grapheme = new Grapheme("a"); + + Assert.That(grapheme.CharValue, Is.EqualTo('a')); + Assert.That(grapheme.StringValue, Is.Null); + Assert.That(grapheme.IsBmp, Is.True); + Assert.That(grapheme.IsSurrogate, Is.False); + Assert.That(grapheme.IsSingleScalarValue, Is.True); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(1)); + } + + /// + /// Tests that the string constructor correctly handles surrogate pairs. + /// + [Test] + public void TestStringConstructorWithSurrogatePair() + { + var grapheme = new Grapheme("πŸ™‚"); // U+1F642 SLIGHTLY SMILING FACE + + Assert.That(grapheme.CharValue, Is.EqualTo('\uFFFD')); // replacement character for high surrogate + Assert.That(grapheme.StringValue, Is.EqualTo("πŸ™‚")); + Assert.That(grapheme.IsBmp, Is.False); + Assert.That(grapheme.IsSurrogate, Is.True); + Assert.That(grapheme.IsSingleScalarValue, Is.True); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(2)); + } + + /// + /// Tests that the string constructor correctly handles complex grapheme clusters. + /// + [Test] + public void TestStringConstructorWithComplexGrapheme() + { + var grapheme = new Grapheme("☝🏽"); // Pointing up with medium skin tone + + Assert.That(grapheme.CharValue, Is.EqualTo('\u261D')); // first char of the sequence + Assert.That(grapheme.StringValue, Is.EqualTo("☝🏽")); + Assert.That(grapheme.IsBmp, Is.False); + Assert.That(grapheme.IsSurrogate, Is.False); + Assert.That(grapheme.IsSingleScalarValue, Is.False); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(3)); + } + + /// + /// Tests that the ReadOnlySpan constructor works correctly. + /// + [Test] + public void TestReadOnlySpanConstructor() + { + var span = "πŸ™‚".AsSpan(); + var grapheme = new Grapheme(span); + + Assert.That(grapheme.CharValue, Is.EqualTo('\uFFFD')); + Assert.That(grapheme.StringValue, Is.EqualTo("πŸ™‚")); + Assert.That(grapheme.IsBmp, Is.False); + Assert.That(grapheme.IsSurrogate, Is.True); + Assert.That(grapheme.IsSingleScalarValue, Is.True); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(2)); + } + + #endregion + + #region Property Tests + + /// + /// Tests that IsBmp correctly identifies Basic Multilingual Plane characters. + /// + [Test] + public void TestIsBmpProperty() + { + Assert.That(new Grapheme('a').IsBmp, Is.True); + Assert.That(new Grapheme('€').IsBmp, Is.True); + Assert.That(new Grapheme('\u2603').IsBmp, Is.True); // snowman + Assert.That(new Grapheme("πŸ™‚").IsBmp, Is.False); + Assert.That(new Grapheme("πŸ‘‹πŸ½").IsBmp, Is.False); + } + + /// + /// Tests that IsSurrogate correctly identifies surrogate pairs. + /// + [Test] + public void TestIsSurrogateProperty() + { + Assert.That(new Grapheme('a').IsSurrogate, Is.False); + Assert.That(new Grapheme("πŸ™‚").IsSurrogate, Is.True); + Assert.That(new Grapheme("πŸ‘‹πŸ½").IsSurrogate, Is.False); // complex grapheme cluster, not a simple surrogate pair + } + + /// + /// Tests that IsSingleScalarValue correctly identifies single Unicode scalar values. + /// + [Test] + public void TestIsSingleScalarValueProperty() + { + Assert.That(new Grapheme('a').IsSingleScalarValue, Is.True); + Assert.That(new Grapheme("πŸ™‚").IsSingleScalarValue, Is.True); + Assert.That(new Grapheme("πŸ‘‹πŸ½").IsSingleScalarValue, Is.False); + Assert.That(new Grapheme("Γ©").IsSingleScalarValue, Is.True); // precomposed + } + + /// + /// Tests that Utf16SequenceLength returns correct values. + /// + [Test] + public void TestUtf16SequenceLengthProperty() + { + Assert.That(new Grapheme('a').Utf16SequenceLength, Is.EqualTo(1)); + Assert.That(new Grapheme("πŸ™‚").Utf16SequenceLength, Is.EqualTo(2)); + Assert.That(new Grapheme("πŸ‘‹πŸ½").Utf16SequenceLength, Is.EqualTo(4)); + } + + #endregion + + #region Method Tests + + /// + /// Tests that IsWhiteSpace correctly identifies whitespace characters. + /// + [Test] + public void TestIsWhiteSpaceMethod() + { + Assert.That(new Grapheme(' ').IsWhiteSpace(), Is.True); + Assert.That(new Grapheme('\t').IsWhiteSpace(), Is.True); + Assert.That(new Grapheme('\n').IsWhiteSpace(), Is.True); + Assert.That(new Grapheme('\r').IsWhiteSpace(), Is.True); + Assert.That(new Grapheme('a').IsWhiteSpace(), Is.False); + Assert.That(new Grapheme("πŸ™‚").IsWhiteSpace(), Is.False); + } + + /// + /// Tests that RemoveLastModifier works correctly with emoji skin tones. + /// + [Test] + public void TestRemoveLastModifierWithSkinTone() + { + var emojiWithSkinTone = new Grapheme("πŸ‘‹πŸ½"); + var baseEmoji = emojiWithSkinTone.RemoveLastModifier(); + + Assert.That(baseEmoji, Is.Not.Null); + Assert.That(baseEmoji?.ToString(), Is.EqualTo("πŸ‘‹")); + } + + /// + /// Tests that RemoveLastModifier returns null for BMP characters. + /// + [Test] + public void TestRemoveLastModifierWithBmpCharacter() + { + var bmpChar = new Grapheme('a'); + var result = bmpChar.RemoveLastModifier(); + + Assert.That(result, Is.Null); + } + + /// + /// Tests that RemoveLastModifier returns null for simple surrogate pairs. + /// + [Test] + public void TestRemoveLastModifierWithSimpleSurrogatePair() + { + var emoji = new Grapheme("πŸ™‚"); + var result = emoji.RemoveLastModifier(); + + Assert.That(result, Is.Null); + } + + /// + /// Tests that ToString returns correct string representation. + /// + [Test] + public void TestToStringMethod() + { + Assert.That(new Grapheme('a').ToString(), Is.EqualTo("a")); + Assert.That(new Grapheme("πŸ™‚").ToString(), Is.EqualTo("πŸ™‚")); + Assert.That(new Grapheme("πŸ‘‹πŸ½").ToString(), Is.EqualTo("πŸ‘‹πŸ½")); + } + + #endregion + + #region Equality Tests + + /// + /// Tests that Equals works correctly for identical graphemes. + /// + [Test] + public void TestEqualsWithIdenticalGraphemes() + { + var grapheme1 = new Grapheme('a'); + var grapheme2 = new Grapheme('a'); + + Assert.That(grapheme1.Equals(grapheme2), Is.True); + Assert.That(grapheme1 == grapheme2, Is.True); + Assert.That(grapheme1 != grapheme2, Is.False); + } + + /// + /// Tests that Equals works correctly for different graphemes. + /// + [Test] + public void TestEqualsWithDifferentGraphemes() + { + var grapheme1 = new Grapheme('a'); + var grapheme2 = new Grapheme('b'); + + Assert.That(grapheme1.Equals(grapheme2), Is.False); + Assert.That(grapheme1 == grapheme2, Is.False); + Assert.That(grapheme1 != grapheme2, Is.True); + } + + /// + /// Tests that Equals works correctly for complex graphemes. + /// + [Test] + public void TestEqualsWithComplexGraphemes() + { + var grapheme1 = new Grapheme("πŸ‘‹πŸ½"); + var grapheme2 = new Grapheme("πŸ‘‹πŸ½"); + var grapheme3 = new Grapheme("πŸ‘‹πŸΎ"); + + Assert.That(grapheme1.Equals(grapheme2), Is.True); + Assert.That(grapheme1 == grapheme2, Is.True); + Assert.That(grapheme1.Equals(grapheme3), Is.False); + Assert.That(grapheme1 == grapheme3, Is.False); + } + + /// + /// Tests that GetHashCode produces consistent results. + /// + [Test] + public void TestGetHashCode() + { + var grapheme1 = new Grapheme('a'); + var grapheme2 = new Grapheme('a'); + var grapheme3 = new Grapheme('b'); + + Assert.That(grapheme1.GetHashCode(), Is.EqualTo(grapheme2.GetHashCode())); + Assert.That(grapheme1.GetHashCode(), Is.Not.EqualTo(grapheme3.GetHashCode())); + } + + #endregion + + #region Operator Tests + + /// + /// Tests explicit conversion from char to Grapheme. + /// + [Test] + public void TestExplicitCastFromChar() + { + const char c = 'a'; + var grapheme = (Grapheme)c; + + Assert.That(grapheme.CharValue, Is.EqualTo('a')); + Assert.That(grapheme.IsBmp, Is.True); + } + + /// + /// Tests explicit conversion from Rune to Grapheme. + /// + [Test] + public void TestExplicitCastFromRune() + { + var rune = new Rune('a'); + var grapheme = (Grapheme)rune; + + Assert.That(grapheme.CharValue, Is.EqualTo('a')); + Assert.That(grapheme.IsBmp, Is.True); + } + + /// + /// Tests explicit conversion from Grapheme to Rune. + /// + [Test] + public void TestExplicitCastToRune() + { + var grapheme = new Grapheme('a'); + var rune = (Rune)grapheme; + + Assert.That(rune.Value, Is.EqualTo('a')); + } + + /// + /// Tests explicit conversion from surrogate pair Grapheme to Rune. + /// + [Test] + public void TestExplicitCastSurrogatePairToRune() + { + var grapheme = new Grapheme("πŸ™‚"); + var rune = (Rune)grapheme; + + Assert.That(rune.ToString(), Is.EqualTo("πŸ™‚")); + } + + #endregion + + #region Static Method Tests + + /// + /// Tests that GetGraphemeEnumerator correctly handles simple text. + /// + [Test] + public void TestGetGraphemeEnumeratorWithSimpleText() + { + const string text = "Hello"; + var graphemes = Grapheme.GetGraphemeEnumerator(text).ToArray(); + + Assert.That(graphemes.Length, Is.EqualTo(5)); + Assert.That(graphemes[0].CharValue, Is.EqualTo('H')); + Assert.That(graphemes[1].CharValue, Is.EqualTo('e')); + Assert.That(graphemes[2].CharValue, Is.EqualTo('l')); + Assert.That(graphemes[3].CharValue, Is.EqualTo('l')); + Assert.That(graphemes[4].CharValue, Is.EqualTo('o')); + } + + /// + /// Tests that GetGraphemeEnumerator correctly handles complex Unicode text. + /// + [Test] + public void TestGetGraphemeEnumeratorWithComplexText() + { + const string text = "Hello πŸ‘‹πŸ½ World!"; + var graphemes = Grapheme.GetGraphemeEnumerator(text).ToArray(); + + Assert.That(graphemes.Length, Is.EqualTo(14)); + Assert.That(graphemes[6].ToString(), Is.EqualTo("πŸ‘‹πŸ½")); + Assert.That(graphemes[6].IsSingleScalarValue, Is.False); + Assert.That(graphemes[6].Utf16SequenceLength, Is.EqualTo(4)); + } + + /// + /// Tests that GetGraphemeEnumerator returns empty enumerable for null or empty strings. + /// + [Test] + public void TestGetGraphemeEnumeratorWithEmptyText() + { + var graphemes1 = Grapheme.GetGraphemeEnumerator(null!).ToArray(); + var graphemes2 = Grapheme.GetGraphemeEnumerator("").ToArray(); + + Assert.That(graphemes1.Length, Is.EqualTo(0)); + Assert.That(graphemes2.Length, Is.EqualTo(0)); + } + + /// + /// Tests that GetGraphemeEnumerator correctly handles text with combining characters. + /// + [Test] + public void TestGetGraphemeEnumeratorWithCombiningCharacters() + { + // "cafΓ©" with combining acute accent (e + Β΄) + const string text = "cafe\u0301"; + var graphemes = Grapheme.GetGraphemeEnumerator(text).ToArray(); + + Assert.That(graphemes.Length, Is.EqualTo(4)); + Assert.That(graphemes[3].ToString(), Is.EqualTo("é")); + Assert.That(graphemes[3].Utf16SequenceLength, Is.EqualTo(2)); + } + + #endregion + + #region Edge Cases and Special Characters + + /// + /// Tests handling of null character. + /// + [Test] + public void TestNullCharacter() + { + var grapheme = new Grapheme('\0'); + + Assert.That(grapheme.CharValue, Is.EqualTo('\0')); + Assert.That(grapheme.IsBmp, Is.True); + Assert.That(grapheme.ToString(), Is.EqualTo("\0")); + } + + /// + /// Tests handling of replacement character. + /// + [Test] + public void TestReplacementCharacter() + { + var grapheme = new Grapheme('\uFFFD'); + + Assert.That(grapheme.CharValue, Is.EqualTo('\uFFFD')); + Assert.That(grapheme.IsBmp, Is.True); + Assert.That(grapheme.ToString(), Is.EqualTo("οΏ½")); + } + + /// + /// Tests that graphemes preserve original string exactly. + /// + [Test] + public void TestStringPreservation() + { + const string original_string = "πŸ‘‹πŸ½"; + var grapheme = new Grapheme(original_string); + + Assert.That(grapheme.ToString(), Is.EqualTo(original_string)); + Assert.That(grapheme.StringValue, Is.EqualTo(original_string)); + } + + /// + /// Tests handling of regional indicator sequences (flags). + /// + [Test] + public void TestRegionalIndicatorSequence() + { + // US flag (πŸ‡ΊπŸ‡Έ) + const string flag = "πŸ‡ΊπŸ‡Έ"; + var grapheme = new Grapheme(flag); + + Assert.That(grapheme.ToString(), Is.EqualTo(flag)); + Assert.That(grapheme.IsBmp, Is.False); + Assert.That(grapheme.IsSingleScalarValue, Is.False); + Assert.That(grapheme.Utf16SequenceLength, Is.EqualTo(4)); + } + + #endregion + } +} diff --git a/osu.Framework.Tests/Text/TextBuilderTest.cs b/osu.Framework.Tests/Text/TextBuilderTest.cs index a0bc358cfc..02b1cbdb4b 100644 --- a/osu.Framework.Tests/Text/TextBuilderTest.cs +++ b/osu.Framework.Tests/Text/TextBuilderTest.cs @@ -567,7 +567,7 @@ public void TestSameCharacterFallsBackWithNoFontName() builder.AddText("a"); - Assert.That(builder.Characters[0].Character, Is.EqualTo('a')); + Assert.That(builder.Characters[0].Character, Is.EqualTo(new Grapheme('a'))); } /// @@ -588,7 +588,7 @@ public void TestSameCharacterFallsBackToDifferentFontWithSameWeight() builder.AddText("a"); - Assert.That(builder.Characters[0].Character, Is.EqualTo('a')); + Assert.That(builder.Characters[0].Character.CharValue, Is.EqualTo('a')); Assert.That(((TestGlyph)builder.Characters[0].Glyph).FontName, Is.EqualTo("test2-Bold")); } @@ -610,7 +610,7 @@ public void TestSameCharacterFallsBackToDifferentFontWithItalics() builder.AddText("a"); - Assert.That(builder.Characters[0].Character, Is.EqualTo('a')); + Assert.That(builder.Characters[0].Character.CharValue, Is.EqualTo('a')); Assert.That(((TestGlyph)builder.Characters[0].Glyph).FontName, Is.EqualTo("test2-Italic")); } @@ -631,7 +631,7 @@ public void TestFallBackCharacterFallsBackWithFontName() builder.AddText("a"); - Assert.That(builder.Characters[0].Character, Is.EqualTo('?')); + Assert.That(builder.Characters[0].Character, Is.EqualTo(new Grapheme('?'))); Assert.That(builder.Characters[0].XOffset, Is.EqualTo(0)); } @@ -652,7 +652,7 @@ public void TestFallBackCharacterFallsBackWithNoFontName() builder.AddText("a"); - Assert.That(builder.Characters[0].Character, Is.EqualTo('?')); + Assert.That(builder.Characters[0].Character, Is.EqualTo(new Grapheme('?'))); Assert.That(builder.Characters[0].XOffset, Is.EqualTo(1)); } @@ -670,6 +670,16 @@ public void TestFailedCharacterLookup() Assert.That(builder.Bounds, Is.EqualTo(Vector2.Zero)); } + [Test] + public void TestSupplementaryCharactersAreOneCharacter() + { + var builder = new TextBuilder(fontStore, normal_font); + + builder.AddText("πŸ™‚"); // πŸ™‚ is U+1F642, which is greater than char.MaxValue (0xFFFF) + + Assert.That(builder.Characters, Has.Count.EqualTo(1)); + } + [TearDown] public void TearDown() { @@ -692,7 +702,7 @@ public TestStore(params GlyphEntry[] glyphs) this.glyphs = glyphs; } - public ITexturedCharacterGlyph Get(string? fontName, char character) + public ITexturedCharacterGlyph Get(string? fontName, Grapheme character) { if (string.IsNullOrEmpty(fontName)) return glyphs.FirstOrDefault(g => g.Glyph.Character == character).Glyph; @@ -700,7 +710,7 @@ public ITexturedCharacterGlyph Get(string? fontName, char character) return glyphs.FirstOrDefault(g => g.Font.FontName.EndsWith(fontName, StringComparison.Ordinal) && g.Glyph.Character == character).Glyph; } - public Task GetAsync(string fontName, char character) => throw new NotImplementedException(); + public Task GetAsync(string fontName, Grapheme character) => throw new NotImplementedException(); } private readonly struct GlyphEntry @@ -724,15 +734,17 @@ public GlyphEntry(FontUsage font, ITexturedCharacterGlyph glyph) public float Width { get; } public float Baseline { get; } public float Height { get; } - public char Character { get; } + public Grapheme Character { get; } public string? FontName { get; } + public bool Coloured => false; private readonly float glyphKerning; - public TestGlyph(char character, float xOffset = 0, float yOffset = 0, float xAdvance = 0, float width = 0, float baseline = 0, float height = 0, float kerning = 0, string? fontName = null) + public TestGlyph(char character, float xOffset = 0, float yOffset = 0, float xAdvance = 0, float width = 0, float baseline = 0, float height = 0, float kerning = 0, + string? fontName = null) { glyphKerning = kerning; - Character = character; + Character = new Grapheme(character); XOffset = xOffset; YOffset = yOffset; XAdvance = xAdvance; diff --git a/osu.Framework.Tests/Visual/Sprites/TestSceneSpriteTextScenarios.cs b/osu.Framework.Tests/Visual/Sprites/TestSceneSpriteTextScenarios.cs index 444c902bd1..0860cb83b7 100644 --- a/osu.Framework.Tests/Visual/Sprites/TestSceneSpriteTextScenarios.cs +++ b/osu.Framework.Tests/Visual/Sprites/TestSceneSpriteTextScenarios.cs @@ -231,6 +231,13 @@ public TestSceneSpriteTextScenarios() Child = new SpriteText { Current = boundString }, }; + Cell(3, 4).Child = new SpriteText + { + RelativeSizeAxes = Axes.Both, + Text = "🐒 ZWJ: πŸ˜Άβ€πŸŒ«οΈ", // 🐒 is U+1F60E, the Zero Width Joiner sequence also has Variation Selector-16 after th emoji glyphs + Font = FrameworkFont.Condensed.With(size: 32), + }; + Scheduler.AddDelayed(() => boundString.Value = $"bindable: {++boundStringValue}", 200, true); } diff --git a/osu.Framework.Tests/Visual/UserInterface/TestSceneTextBox.cs b/osu.Framework.Tests/Visual/UserInterface/TestSceneTextBox.cs index c8e121376e..c5845a9235 100644 --- a/osu.Framework.Tests/Visual/UserInterface/TestSceneTextBox.cs +++ b/osu.Framework.Tests/Visual/UserInterface/TestSceneTextBox.cs @@ -13,6 +13,7 @@ using osu.Framework.Graphics.UserInterface; using osu.Framework.Input; using osu.Framework.Testing; +using osu.Framework.Text; using osu.Framework.Utils; using osuTK; using osuTK.Input; @@ -1119,6 +1120,30 @@ public void TestTabbing() AddAssert("first textbox focused", () => textBoxes[0].HasFocus); } + [Test] + public void TestEmoji() + { + TestTextBox textBox = null; + + AddStep("add text box", () => + { + textBoxes.Add(textBox = new TestTextBox + { + Size = new Vector2(300, 40), + Text = "πŸ™‚", + }); + }); + + AddAssert("only one sprite text", () => textBox.TextFlow.FlowingChildren.ToList(), () => Has.Count.EqualTo(1)); + + AddStep("add text box with combined emoji", () => + { + textBox.Text = "βœ‹πŸΏπŸ»β€β„οΈ"; // βœ‹πŸΏ = 270B(raised hand) + 1F3FF(dark skin tone) , πŸ»β€β„οΈ = 1F43B(bear) + 200D(ZWJ) + 2744(snowflake) + FE0F(VS-16) + }); + + AddAssert("should have 2 sprite texts", () => textBox.TextFlow.FlowingChildren.ToList(), () => Has.Count.EqualTo(2)); + } + private void prependString(InsertableTextBox textBox, string text) { InputManager.Keys(PlatformAction.MoveBackwardLine); @@ -1161,13 +1186,13 @@ public partial class InsertableTextBox : BasicTextBox private partial class CustomTextBox : BasicTextBox { - protected override Drawable GetDrawableCharacter(char c) => new ScalingText(c, FontSize); + protected override Drawable GetDrawableCharacter(Grapheme c) => new ScalingText(c, FontSize); private partial class ScalingText : CompositeDrawable { private readonly SpriteText text; - public ScalingText(char c, float textSize) + public ScalingText(Grapheme c, float textSize) { AddInternal(text = new SpriteText { @@ -1240,5 +1265,10 @@ private partial class PaddedTextBox : BasicTextBox public bool TextContainerTransformsFinished => TextContainer.LatestTransformEndTime == TextContainer.TransformStartTime; } + + private partial class TestTextBox : BasicTextBox + { + public new FillFlowContainer TextFlow => base.TextFlow; + } } } diff --git a/osu.Framework/Graphics/Sprites/IconUsage.cs b/osu.Framework/Graphics/Sprites/IconUsage.cs index b9f7e35d9d..5157f84681 100644 --- a/osu.Framework/Graphics/Sprites/IconUsage.cs +++ b/osu.Framework/Graphics/Sprites/IconUsage.cs @@ -45,7 +45,7 @@ namespace osu.Framework.Graphics.Sprites /// /// Creates an instance of using the specified font , font and a value indicating whether the used font is italic or not. /// - /// /// The icon. + /// The icon. /// The font family name. /// The font weight. public IconUsage(char icon, [CanBeNull] string family = null, [CanBeNull] string weight = null) diff --git a/osu.Framework/Graphics/Sprites/SpriteText.cs b/osu.Framework/Graphics/Sprites/SpriteText.cs index dce188d6b6..4bdd4a4d15 100644 --- a/osu.Framework/Graphics/Sprites/SpriteText.cs +++ b/osu.Framework/Graphics/Sprites/SpriteText.cs @@ -72,7 +72,7 @@ private void load(ShaderManager shaders) TextureShader = shaders.Load(VertexShaderDescriptor.TEXTURE_2, FragmentShaderDescriptor.TEXTURE); // Pre-cache the characters in the texture store - foreach (char character in localisedText.Value) + foreach (var character in Grapheme.GetGraphemeEnumerator(localisedText.Value)) { _ = store.Get(font.FontName, character) ?? store.Get(null, character); } diff --git a/osu.Framework/Graphics/Sprites/SpriteText_DrawNode.cs b/osu.Framework/Graphics/Sprites/SpriteText_DrawNode.cs index 5657a4c700..1c2cd2b52d 100644 --- a/osu.Framework/Graphics/Sprites/SpriteText_DrawNode.cs +++ b/osu.Framework/Graphics/Sprites/SpriteText_DrawNode.cs @@ -75,7 +75,12 @@ protected override void Draw(IRenderer renderer) finalShadowColour, inflationPercentage: parts[i].InflationPercentage); } - renderer.DrawQuad(parts[i].Texture, parts[i].DrawQuad, DrawColourInfo.Colour, inflationPercentage: parts[i].InflationPercentage); + renderer.DrawQuad( + parts[i].Texture, + parts[i].DrawQuad, + // If the character has a coloured texture, use white colour to preserve the original texture colour. + parts[i].Coloured ? Colour4.White : DrawColourInfo.Colour, + inflationPercentage: parts[i].InflationPercentage); } UnbindTextureShader(renderer); @@ -106,7 +111,8 @@ private void updateScreenSpaceCharacters() InflationPercentage = new Vector2( character.DrawRectangle.Size.X == 0 ? 0 : inflationAmount.X / character.DrawRectangle.Size.X, character.DrawRectangle.Size.Y == 0 ? 0 : inflationAmount.Y / character.DrawRectangle.Size.Y), - Texture = character.Texture + Texture = character.Texture, + Coloured = character.Coloured }); } } @@ -131,6 +137,8 @@ internal struct ScreenSpaceCharacterPart /// The texture to draw the character with. /// public Texture Texture; + + public bool Coloured; } } } diff --git a/osu.Framework/Graphics/UserInterface/BasicTextBox.cs b/osu.Framework/Graphics/UserInterface/BasicTextBox.cs index 381699178e..ad05479a25 100644 --- a/osu.Framework/Graphics/UserInterface/BasicTextBox.cs +++ b/osu.Framework/Graphics/UserInterface/BasicTextBox.cs @@ -6,6 +6,7 @@ using osu.Framework.Graphics.Shapes; using osu.Framework.Graphics.Sprites; using osu.Framework.Input.Events; +using osu.Framework.Text; using osuTK; using osuTK.Graphics; @@ -93,7 +94,7 @@ protected override void OnFocus(FocusEvent e) background.FadeColour(BackgroundFocused, 200, Easing.Out); } - protected override Drawable GetDrawableCharacter(char c) => new FallingDownContainer + protected override Drawable GetDrawableCharacter(Grapheme c) => new FallingDownContainer { AutoSizeAxes = Axes.Both, Child = new SpriteText { Text = c.ToString(), Font = FrameworkFont.Condensed.With(size: FontSize) } diff --git a/osu.Framework/Graphics/UserInterface/TextBox.cs b/osu.Framework/Graphics/UserInterface/TextBox.cs index a646d38c78..0f9ad05c7c 100644 --- a/osu.Framework/Graphics/UserInterface/TextBox.cs +++ b/osu.Framework/Graphics/UserInterface/TextBox.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Globalization; using System.Linq; -using System.Text; using JetBrains.Annotations; using osu.Framework.Allocation; using osu.Framework.Bindables; @@ -24,6 +23,7 @@ using osu.Framework.Input.Events; using osu.Framework.Localisation; using osu.Framework.Platform; +using osu.Framework.Text; using osu.Framework.Threading; using osuTK; using osuTK.Input; @@ -48,7 +48,8 @@ public abstract partial class TextBox : TabbableContainer, IHasCurrentValue 5; /// - /// Maximum allowed length of text. + /// Maximum allowed length of text in UTF-16 code units (not grapheme clusters). + /// Note that complex Unicode characters like emoji with modifiers may count as multiple units. /// /// Any input beyond this limit will be dropped and then will be called. public int? LengthLimit; @@ -443,7 +444,7 @@ public bool SelectAll() return false; selectionStart = 0; - selectionEnd = text.Length; + selectionEnd = graphemes.Count; cursorAndLayout.Invalidate(); return true; } @@ -540,10 +541,19 @@ private static int findNextWord(string text, int position, int direction) return position; } - // Currently only single line is supported and line length and text length are the same. - protected int GetBackwardLineAmount() => -text.Length; + /// + /// Gets the amount to move cursor backward to reach the beginning of the line. + /// Currently only single line is supported and line length equals grapheme count. + /// + /// Negative number representing backward movement in grapheme units. + protected int GetBackwardLineAmount() => -graphemes.Count; - protected int GetForwardLineAmount() => text.Length; + /// + /// Gets the amount to move cursor forward to reach the end of the line. + /// Currently only single line is supported and line length equals grapheme count. + /// + /// Positive number representing forward movement in grapheme units. + protected int GetForwardLineAmount() => graphemes.Count; /// /// Move the current cursor by the signed . @@ -573,7 +583,7 @@ protected void ExpandSelectionBy(int amount) protected void DeleteBy(int amount) { if (selectionLength == 0) - selectionEnd = Math.Clamp(selectionStart + amount, 0, text.Length); + selectionEnd = Math.Clamp(selectionStart + amount, 0, graphemes.Count); if (hasSelection) { @@ -646,7 +656,7 @@ private void updateCursorAndLayout() Placeholder.Font = Placeholder.Font.With(size: FontSize); float cursorPos = 0; - if (text.Length > 0) + if (graphemes.Count > 0) cursorPos = getPositionAt(selectionLeft); float cursorPosEnd = getPositionAt(selectionEnd); @@ -713,7 +723,7 @@ private float getPositionAt(int index) { if (index > 0) { - if (index < text.Length) + if (index < graphemes.Count) return TextFlow.Children[index].DrawPosition.X + TextFlow.DrawPosition.X; var d = TextFlow.Children[index - 1]; @@ -740,13 +750,34 @@ private int getCharacterClosestTo(Vector2 pos) return i; } + /// + /// Start position of the selection, measured in grapheme units (not UTF-16 code units). + /// private int selectionStart; + + /// + /// End position of the selection, measured in grapheme units (not UTF-16 code units). + /// private int selectionEnd; + /// + /// Length of the current selection in grapheme units. + /// private int selectionLength => Math.Abs(selectionEnd - selectionStart); + + /// + /// Whether there is currently an active text selection. + /// private bool hasSelection => selectionLength > 0; + /// + /// Left boundary of the selection (minimum of start and end) in grapheme units. + /// private int selectionLeft => Math.Min(selectionStart, selectionEnd); + + /// + /// Right boundary of the selection (maximum of start and end) in grapheme units. + /// private int selectionRight => Math.Max(selectionStart, selectionEnd); private readonly Cached cursorAndLayout = new Cached(); @@ -759,7 +790,7 @@ private void moveSelection(int offset, bool expand) int oldEnd = selectionEnd; if (expand) - selectionEnd = Math.Clamp(selectionEnd + offset, 0, text.Length); + selectionEnd = Math.Clamp(selectionEnd + offset, 0, graphemes.Count); else { if (hasSelection && Math.Abs(offset) <= 1) @@ -771,7 +802,7 @@ private void moveSelection(int offset, bool expand) selectionEnd = selectionStart = selectionLeft; } else - selectionEnd = selectionStart = Math.Clamp((offset > 0 ? selectionRight : selectionLeft) + offset, 0, text.Length); + selectionEnd = selectionStart = Math.Clamp((offset > 0 ? selectionRight : selectionLeft) + offset, 0, graphemes.Count); } if (oldStart != selectionStart || oldEnd != selectionEnd) @@ -836,6 +867,7 @@ private string removeCharacters(int number = 1) return string.Empty; int removeStart = Math.Clamp(selectionRight - number, 0, selectionRight); + int removeStartInText = graphemes[..removeStart].Sum(g => g.Utf16SequenceLength); int removeCount = selectionRight - removeStart; if (removeCount == 0) @@ -858,9 +890,11 @@ private string removeCharacters(int number = 1) d.Expire(); } - string removedText = text.Substring(removeStart, removeCount); + int removeCountInText = graphemes[removeStart..(removeStart + removeCount)].Sum(g => g.Utf16SequenceLength); - text = text.Remove(removeStart, removeCount); + graphemes.RemoveRange(removeStart, removeCount); + string removedText = text.Substring(removeStartInText, removeCountInText); + text = text.Remove(removeStartInText, removeCountInText); // Reorder characters depth after removal to avoid ordering issues with newly added characters. for (int i = removeStart; i < TextFlow.Count; i++) @@ -880,12 +914,12 @@ private string removeCharacters(int number = 1) /// /// The character that this should represent. /// A that represents the character - protected virtual Drawable GetDrawableCharacter(char c) => new SpriteText { Text = c.ToString(), Font = new FontUsage(size: FontSize) }; + protected virtual Drawable GetDrawableCharacter(Grapheme c) => new SpriteText { Text = c.ToString(), Font = new FontUsage(size: FontSize) }; - protected virtual Drawable AddCharacterToFlow(char c) + protected virtual Drawable AddCharacterToFlow(Grapheme c) { if (InputProperties.Type.IsPassword()) - c = MaskCharacter; + c = new Grapheme(MaskCharacter); // Remove all characters to the right and store them in a local list, // such that their depth can be updated. @@ -945,9 +979,9 @@ private void insertString(string value, Action drawableCreationParamet bool beganChange = beginTextChange(); - foreach (char c in value) + foreach (var c in Grapheme.GetGraphemeEnumerator(value)) { - if (!canAddCharacter(c)) + if (!canAddCharacter(c.CharValue)) { NotifyInputError(); continue; @@ -956,7 +990,7 @@ private void insertString(string value, Action drawableCreationParamet if (hasSelection) removeSelection(); - if (text.Length + 1 > LengthLimit) + if (text.Length + c.Utf16SequenceLength > LengthLimit) { NotifyInputError(); break; @@ -967,7 +1001,8 @@ private void insertString(string value, Action drawableCreationParamet drawable.Show(); drawableCreationParameters?.Invoke(drawable); - text = text.Insert(selectionLeft, c.ToString()); + text = text.Insert(graphemes[..selectionLeft].Sum(g => g.Utf16SequenceLength), c.ToString()); + graphemes.Insert(selectionLeft, c); selectionStart = selectionEnd = selectionLeft + 1; ignoreOngoingDragSelection = true; @@ -1056,8 +1091,8 @@ private void onTextDeselected((int start, int end) lastSelectionBounds) /// Invoked whenever the IME composition has changed. /// /// The current text of the composition. - /// The number of characters that have been replaced by new ones. - /// The number of characters that have replaced the old ones. + /// The number of graphemes that have been replaced by new ones. + /// The number of graphemes that have replaced the old ones. /// Whether the selection/caret has moved. protected virtual void OnImeComposition(string newComposition, int removedTextLength, int addedTextLength, bool selectionMoved) { @@ -1126,6 +1161,15 @@ public Bindable Current private string text = string.Empty; + /// + /// The graphemes that make up the text. This list represents the logical characters as perceived by users, + /// and is used for cursor positioning, selection, and navigation. Each may consist + /// of multiple UTF-16 code units in the string (e.g., emoji with skin tone modifiers). + /// The count of this list represents the number of user-perceived characters, while .Length + /// represents the number of UTF-16 code units. + /// + private readonly List graphemes = new List(); + public virtual string Text { get => text; @@ -1161,6 +1205,7 @@ private void setText(string value) TextFlow?.Clear(); text = string.Empty; + graphemes.Clear(); // insert string and fast forward any transforms (generally when replacing the full content of a textbox we don't want any kind of fade etc.). insertString(value, d => d.FinishTransforms()); @@ -1169,7 +1214,7 @@ private void setText(string value) cursorAndLayout.Invalidate(); } - public string SelectedText => hasSelection ? Text.Substring(selectionLeft, selectionLength) : string.Empty; + public string SelectedText => hasSelection ? string.Concat(graphemes[selectionLeft..(selectionLeft + selectionLength)]) : string.Empty; /// /// Whether s should be blocked because of recent text input from a . @@ -1319,13 +1364,13 @@ protected override void OnDrag(DragEvent e) if (getCharacterClosestTo(e.MousePosition) > doubleClickWord[1]) { selectionStart = doubleClickWord[0]; - selectionEnd = findSeparatorIndex(text, getCharacterClosestTo(e.MousePosition) - 1, 1); - selectionEnd = selectionEnd >= 0 ? selectionEnd : text.Length; + selectionEnd = findSeparatorIndex(graphemes, getCharacterClosestTo(e.MousePosition) - 1, 1); + selectionEnd = selectionEnd >= 0 ? selectionEnd : graphemes.Count; } else if (getCharacterClosestTo(e.MousePosition) < doubleClickWord[0]) { selectionStart = doubleClickWord[1]; - selectionEnd = findSeparatorIndex(text, getCharacterClosestTo(e.MousePosition), -1); + selectionEnd = findSeparatorIndex(graphemes, getCharacterClosestTo(e.MousePosition), -1); selectionEnd = selectionEnd >= 0 ? selectionEnd + 1 : 0; } else @@ -1337,7 +1382,7 @@ protected override void OnDrag(DragEvent e) } else { - if (text.Length == 0) return; + if (graphemes.Count == 0) return; selectionEnd = getCharacterClosestTo(e.MousePosition); if (hasSelection) @@ -1355,22 +1400,22 @@ protected override bool OnDoubleClick(DoubleClickEvent e) var lastSelectionBounds = getTextSelectionBounds(); - if (text.Length == 0) return true; + if (graphemes.Count == 0) return true; if (AllowClipboardExport) { - int hover = Math.Min(text.Length - 1, getCharacterClosestTo(e.MousePosition)); + int hover = Math.Min(graphemes.Count - 1, getCharacterClosestTo(e.MousePosition)); - int lastSeparator = findSeparatorIndex(text, hover, -1); - int nextSeparator = findSeparatorIndex(text, hover, 1); + int lastSeparator = findSeparatorIndex(graphemes, hover, -1); + int nextSeparator = findSeparatorIndex(graphemes, hover, 1); selectionStart = lastSeparator >= 0 ? lastSeparator + 1 : 0; - selectionEnd = nextSeparator >= 0 ? nextSeparator : text.Length; + selectionEnd = nextSeparator >= 0 ? nextSeparator : graphemes.Count; } else { selectionStart = 0; - selectionEnd = text.Length; + selectionEnd = graphemes.Count; } //in order to keep the home word selected @@ -1383,13 +1428,13 @@ protected override bool OnDoubleClick(DoubleClickEvent e) return true; } - private static int findSeparatorIndex(string input, int searchPos, int direction) + private static int findSeparatorIndex(List input, int searchPos, int direction) { - bool isLetterOrDigit = char.IsLetterOrDigit(input[searchPos]); + bool isLetterOrDigit = char.IsLetterOrDigit(input[searchPos].CharValue); - for (int i = searchPos; i >= 0 && i < input.Length; i += direction) + for (int i = searchPos; i >= 0 && i < input.Count; i += direction) { - if (char.IsLetterOrDigit(input[i]) != isLetterOrDigit) + if (char.IsLetterOrDigit(input[i].CharValue) != isLetterOrDigit) return i; } @@ -1533,7 +1578,7 @@ private void handleImeResult(string result) { imeCompositionScheduler.Add(() => { - onImeComposition(result, result.Length, 0, false); + onImeComposition(result, Grapheme.GetGraphemeEnumerator(result).Count(), 0, false); onImeResult(true, true); }); } @@ -1544,9 +1589,9 @@ private void handleImeResult(string result) /// /// Characters matched from the beginning will not match from the end. /// - private void matchBeginningEnd(string a, string b, out int matchBeginning, out int matchEnd) + private void matchBeginningEnd(List a, List b, out int matchBeginning, out int matchEnd) { - int minLength = Math.Min(a.Length, b.Length); + int minLength = Math.Min(a.Count, b.Count); matchBeginning = 0; @@ -1580,13 +1625,13 @@ private bool sanitizeComposition(ref string composition, ref int selectionStart, // remove characters that can't be added. - var builder = new StringBuilder(composition); + var compositionGraphemes = Grapheme.GetGraphemeEnumerator(composition).ToList(); - for (int index = 0; index < builder.Length; index++) + for (int index = 0; index < compositionGraphemes.Count; index++) { - if (!canAddCharacter(builder[index])) + if (!canAddCharacter(compositionGraphemes[index].CharValue)) { - builder.Remove(index, 1); + compositionGraphemes.RemoveAt(index); sanitized = true; if (index < selectionStart) @@ -1604,15 +1649,16 @@ private bool sanitizeComposition(ref string composition, ref int selectionStart, } if (sanitized) - composition = builder.ToString(); + composition = string.Concat(compositionGraphemes); // trim composition if goes beyond the LengthLimit. - int lengthWithoutComposition = text.Length - imeCompositionLength; + int lengthWithoutComposition = graphemes[..^imeCompositionLength].Sum(g => g.Utf16SequenceLength); if (lengthWithoutComposition + composition.Length > LengthLimit) { - composition = composition.Substring(0, (int)LengthLimit - lengthWithoutComposition); + composition = composition[..((int)LengthLimit - lengthWithoutComposition)]; + compositionGraphemes = Grapheme.GetGraphemeEnumerator(composition).ToList(); sanitized = true; } @@ -1620,15 +1666,15 @@ private bool sanitizeComposition(ref string composition, ref int selectionStart, // the selection could be out of bounds if it was trimmed by the above, // or if the platform-native composition event was ill-formed. - if (selectionStart > composition.Length) + if (selectionStart > compositionGraphemes.Count) { - selectionStart = composition.Length; + selectionStart = compositionGraphemes.Count; sanitized = true; } - if (selectionStart + selectionLength > composition.Length) + if (selectionStart + selectionLength > compositionGraphemes.Count) { - selectionLength = composition.Length - selectionStart; + selectionLength = compositionGraphemes.Count - selectionStart; sanitized = true; } @@ -1642,7 +1688,7 @@ private bool sanitizeComposition(ref string composition, ref int selectionStart, private readonly List imeCompositionDrawables = new List(); /// - /// Length of the current IME composition. + /// The number of graphemes in the current IME composition. /// /// A length of 0 means that IME composition isn't active. private int imeCompositionLength => imeCompositionDrawables.Count; @@ -1711,12 +1757,13 @@ private void onImeComposition(string newComposition, int newSelectionStart, int NotifyInputError(); } - string oldComposition = text.Substring(imeCompositionStart, imeCompositionLength); + List newCompositionGraphemes = Grapheme.GetGraphemeEnumerator(newComposition).ToList(); + List oldCompositionGraphemes = graphemes[imeCompositionStart..(imeCompositionStart + imeCompositionLength)].ToList(); - matchBeginningEnd(oldComposition, newComposition, out int matchBeginning, out int matchEnd); + matchBeginningEnd(oldCompositionGraphemes, newCompositionGraphemes, out int matchBeginning, out int matchEnd); // how many characters have been removed, starting from `matchBeginning` - int removeCount = oldComposition.Length - matchEnd - matchBeginning; + int removeCount = oldCompositionGraphemes.Count - matchEnd - matchBeginning; // remove the characters that don't match if (removeCount > 0) @@ -1729,11 +1776,11 @@ private void onImeComposition(string newComposition, int newSelectionStart, int } // how many characters have been added, starting from `matchBeginning` - int addCount = newComposition.Length - matchEnd - matchBeginning; + int addCount = newCompositionGraphemes.Count - matchEnd - matchBeginning; if (addCount > 0) { - string addedText = newComposition.Substring(matchBeginning, addCount); + string addedText = string.Concat(newCompositionGraphemes[matchBeginning..(matchBeginning + addCount)]); // set up selection for `insertString` selectionStart = selectionEnd = imeCompositionStart + matchBeginning; @@ -1777,7 +1824,7 @@ private void onImeResult(bool userEvent, bool successful) // move the cursor to end of finalized composition. selectionStart = selectionEnd = imeCompositionStart + imeCompositionLength; - if (userEvent) OnImeResult(text.Substring(imeCompositionStart, imeCompositionLength), successful); + if (userEvent) OnImeResult(string.Concat(graphemes[imeCompositionStart..(imeCompositionStart + imeCompositionLength)]), successful); } imeCompositionDrawables.Clear(); diff --git a/osu.Framework/IO/Stores/FontStore.cs b/osu.Framework/IO/Stores/FontStore.cs index 3991c84cce..823ac0e7a6 100644 --- a/osu.Framework/IO/Stores/FontStore.cs +++ b/osu.Framework/IO/Stores/FontStore.cs @@ -28,7 +28,7 @@ public class FontStore : TextureStore, ITexturedGlyphLookupStore /// A local cache to avoid string allocation overhead. Can be changed to (string,char)=>string if this ever becomes an issue, /// but as long as we directly inherit this is a slight optimisation. /// - private readonly ConcurrentDictionary<(string, char), ITexturedCharacterGlyph> namespacedGlyphCache = new ConcurrentDictionary<(string, char), ITexturedCharacterGlyph>(); + private readonly ConcurrentDictionary<(string, Grapheme), ITexturedCharacterGlyph> namespacedGlyphCache = new ConcurrentDictionary<(string, Grapheme), ITexturedCharacterGlyph>(); /// /// Construct a font store to be added to a parent font store via . @@ -138,7 +138,7 @@ public override void RemoveStore(ITextureStore store) base.RemoveStore(store); } - public ITexturedCharacterGlyph Get(string fontName, char character) + public ITexturedCharacterGlyph Get(string fontName, Grapheme character) { var key = (fontName, character); @@ -164,6 +164,13 @@ public ITexturedCharacterGlyph Get(string fontName, char character) return namespacedGlyphCache[key] = null; } - public Task GetAsync(string fontName, char character) => Task.Run(() => Get(fontName, character)); + /// + /// Retrieves a glyph from the store. + /// This is a convenience method that converts the character to a and calls . + /// Useful for writing tests. + /// + public ITexturedCharacterGlyph Get(string fontName, char character) => Get(fontName, new Grapheme(character)); + + public Task GetAsync(string fontName, Grapheme character) => Task.Run(() => Get(fontName, character)); } } diff --git a/osu.Framework/IO/Stores/GlyphStore.cs b/osu.Framework/IO/Stores/GlyphStore.cs index 3688f8b52c..4efe617b04 100644 --- a/osu.Framework/IO/Stores/GlyphStore.cs +++ b/osu.Framework/IO/Stores/GlyphStore.cs @@ -9,6 +9,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; @@ -101,7 +102,7 @@ public Task LoadFontAsync() => fontLoadTask ??= Task.Factory.StartNew(() => } }, TaskCreationOptions.PreferFairness); - public bool HasGlyph(char c) => Font?.Characters.ContainsKey(c) == true; + public bool HasGlyph(Grapheme c) => c.IsSingleScalarValue && Font?.Characters.ContainsKey(((Rune)c).Value) == true; protected virtual TextureUpload GetPageImage(int page) { @@ -118,33 +119,58 @@ protected string GetFilenameForPage(int page) return $@"{AssetName}_{page.ToString().PadLeft((Font.Pages.Count - 1).ToString().Length, '0')}.png"; } - public CharacterGlyph Get(char character) + public CharacterGlyph Get(Grapheme character) { if (Font == null) return null; Debug.Assert(Baseline != null); - var bmCharacter = Font.GetCharacter(character); + var bmCharacter = Font.GetCharacter(character.CharValue); + + Debug.Assert(bmCharacter != null); return new CharacterGlyph(character, bmCharacter.XOffset, bmCharacter.YOffset, bmCharacter.XAdvance, Baseline.Value, this); } - public int GetKerning(char left, char right) => Font?.GetKerningAmount(left, right) ?? 0; + /// + /// This is a convenience method that converts the character to a and calls . + /// + /// The character to retrieve. + public CharacterGlyph Get(char character) + { + return Get(new Grapheme(character)); + } + + public int GetKerning(Grapheme left, Grapheme right) => Font?.GetKerningAmount(left.CharValue, right.CharValue) ?? 0; Task IResourceStore.GetAsync(string name, CancellationToken cancellationToken) => - Task.Run(() => ((IGlyphStore)this).Get(name[0]), cancellationToken); + Task.Run(() => ((IGlyphStore)this).Get(new Grapheme(name)), cancellationToken); - CharacterGlyph IResourceStore.Get(string name) => Get(name[0]); + CharacterGlyph IResourceStore.Get(string name) => Get(new Grapheme(name)); public TextureUpload Get(string name) { if (Font == null) return null; - if (name.Length > 1 && !name.StartsWith($@"{FontName}/", StringComparison.Ordinal)) - return null; + Grapheme grapheme; + + // name is expected to be in the format "{Grapheme}" or "Font:{FontName}/{Grapheme}" + // this is a shorthand to check if there is a font name in the lookup + if (name.Length > 1) + { + // if FontName does not match, return null. + if (!name.StartsWith($@"{FontName}/", StringComparison.Ordinal)) + return null; + + grapheme = new Grapheme(name.AsSpan(FontName.Length + 1)); + } + else + { + grapheme = new Grapheme(name); + } - return Font.Characters.TryGetValue(name.Last(), out Character c) ? LoadCharacter(c) : null; + return Font.Characters.TryGetValue(grapheme.CharValue, out Character c) ? LoadCharacter(c) : null; } public virtual async Task GetAsync(string name, CancellationToken cancellationToken = default) @@ -152,11 +178,9 @@ public virtual async Task GetAsync(string name, CancellationToken if (name.Length > 1 && !name.StartsWith($@"{FontName}/", StringComparison.Ordinal)) return null; - var bmFont = await completionSource.Task.ConfigureAwait(false); + await completionSource.Task.ConfigureAwait(false); - return bmFont.Characters.TryGetValue(name.Last(), out Character c) - ? LoadCharacter(c) - : null; + return Get(name); } protected int LoadedGlyphCount; diff --git a/osu.Framework/IO/Stores/IGlyphStore.cs b/osu.Framework/IO/Stores/IGlyphStore.cs index ca0c06ac31..23b344316d 100644 --- a/osu.Framework/IO/Stores/IGlyphStore.cs +++ b/osu.Framework/IO/Stores/IGlyphStore.cs @@ -29,14 +29,14 @@ public interface IGlyphStore : IResourceStore /// /// Whether a glyph exists for the specified character in this store. /// - bool HasGlyph(char c); + bool HasGlyph(Grapheme c); /// /// Retrieves a that contains associated spacing information for a character. /// /// The character to retrieve the for. /// The containing associated spacing information for . - CharacterGlyph? Get(char character); + CharacterGlyph? Get(Grapheme character); /// /// Retrieves the kerning for a pair of characters. @@ -44,6 +44,6 @@ public interface IGlyphStore : IResourceStore /// The character to the left. /// The character to the right. /// The kerning. - int GetKerning(char left, char right); + int GetKerning(Grapheme left, Grapheme right); } } diff --git a/osu.Framework/Text/CharacterGlyph.cs b/osu.Framework/Text/CharacterGlyph.cs index ec3595d8de..abcd1ff5af 100644 --- a/osu.Framework/Text/CharacterGlyph.cs +++ b/osu.Framework/Text/CharacterGlyph.cs @@ -12,13 +12,16 @@ public sealed class CharacterGlyph : ICharacterGlyph public float YOffset { get; } public float XAdvance { get; } public float Baseline { get; } - public char Character { get; } + public Grapheme Character { get; } - private readonly IGlyphStore? containingStore; + /// + /// The glyph store that contains this character. + /// + public IGlyphStore? ContainingStore { get; } - public CharacterGlyph(char character, float xOffset, float yOffset, float xAdvance, float baseline, IGlyphStore? containingStore) + public CharacterGlyph(Grapheme character, float xOffset, float yOffset, float xAdvance, float baseline, IGlyphStore? containingStore) { - this.containingStore = containingStore; + ContainingStore = containingStore; Character = character; XOffset = xOffset; @@ -30,6 +33,6 @@ public CharacterGlyph(char character, float xOffset, float yOffset, float xAdvan [MethodImpl(MethodImplOptions.AggressiveInlining)] public float GetKerning(T lastGlyph) where T : ICharacterGlyph - => containingStore?.GetKerning(lastGlyph.Character, Character) ?? 0; + => ContainingStore?.GetKerning(lastGlyph.Character, Character) ?? 0; } } diff --git a/osu.Framework/Text/Grapheme.cs b/osu.Framework/Text/Grapheme.cs new file mode 100644 index 0000000000..f96ccebeeb --- /dev/null +++ b/osu.Framework/Text/Grapheme.cs @@ -0,0 +1,259 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace osu.Framework.Text +{ + /// + /// Represents a single grapheme cluster in Unicode text. A grapheme cluster is what users perceive as a single character, + /// which can be composed of one or more Unicode code points. This struct efficiently handles both simple BMP characters + /// and complex multi-character sequences like emoji with skin tone modifiers or characters with combining marks. + /// + /// + /// This implementation optimizes for the common case of single BMP characters by storing them directly as a char, + /// while falling back to string storage for more complex grapheme clusters. This approach minimizes memory allocations + /// and provides better performance for typical text processing scenarios. + /// + /// Comparison of , , and : + /// + /// string text = "Hello πŸ‘‹πŸ½ cafΓ©!"; + /// + /// // Different counting methods: + /// Console.WriteLine($"String.Length: {text.Length}"); // 13 (UTF-16 code units) + /// Console.WriteLine($"Rune count: {text.EnumerateRunes().Count()}"); // 11 (Unicode code points) + /// Console.WriteLine($"Grapheme count: {Grapheme.GetGraphemeEnumerator(text).Count()}"); // 10 (user-perceived characters) + /// + /// // Why the difference? + /// // 'πŸ‘‹πŸ½' = 1 grapheme, but 2 runes (πŸ‘‹ + 🏽), and 4 UTF-16 code units + /// // 'Γ©' might be 1 grapheme, 1 or 2 runes (precomposed vs base+combining), 1 or 2 UTF-16 units + /// + /// // Processing each type: + /// foreach (char c in text) { /* processes UTF-16 code units */ } + /// foreach (Rune r in text.EnumerateRunes()) { /* processes Unicode code points */ } + /// foreach (Grapheme g in Grapheme.GetGraphemeEnumerator(text)) { /* processes user-perceived characters */ } + /// + /// + public readonly struct Grapheme : IEquatable + { + /// + /// String representation for multi-character grapheme clusters (null for single BMP characters). + /// Used for complex Unicode sequences like surrogate pairs, emoji with modifiers, or combining character sequences. + /// + private string? str { get; init; } + + /// + /// Single character value for BMP characters, or the first character/replacement character for complex sequences. + /// For surrogate pairs, this contains the replacement character (U+FFFD) to avoid invalid char values. + /// + private char c { get; init; } + + /// + /// Gets the character value for single BMP characters. + /// For multi-character sequences, returns the first character or a replacement character for surrogate pairs. + /// + public char CharValue => c; + + /// + /// Gets the string representation for multi-character grapheme clusters. + /// Returns null for single BMP characters that are stored efficiently as a char. + /// + public string? StringValue => str; + + /// + /// Gets the length of the UTF-16 sequence representing this grapheme. + /// Returns 1 for single BMP characters, or the actual string length for complex sequences. + /// + public int Utf16SequenceLength => str?.Length ?? 1; + + /// + /// Gets a value indicating whether this grapheme represents a single character in the Basic Multilingual Plane (BMP). + /// BMP characters can be represented by a single 16-bit char value (U+0000 to U+FFFF). + /// + public bool IsBmp => str == null; + + /// + /// Gets a value indicating whether this grapheme represents a Unicode surrogate pair. + /// Surrogate pairs are used to represent characters outside the BMP (U+10000 to U+10FFFF) using two 16-bit values. + /// + public bool IsSurrogate => str != null && str.Length == 2 && str[0] >= 0xD800U && str[0] <= 0xDBFFU && str[1] >= 0xDC00U && str[1] <= 0xDFFFU; + + /// + /// Gets a value indicating whether this grapheme represents a single Unicode scalar value. + /// This includes both BMP characters and surrogate pairs, but excludes complex grapheme clusters + /// with multiple scalar values (like emoji with modifiers or combining characters). + /// + public bool IsSingleScalarValue => IsBmp || IsSurrogate; + + /// + /// Determines whether this grapheme represents a whitespace character. + /// Currently only supports detection for single BMP characters. + /// + /// true if the grapheme is a whitespace character; otherwise, false. + public bool IsWhiteSpace() + { + if (IsBmp) + return char.IsWhiteSpace(c); + + // We don't support whitespace detection for multi-character sequences + return false; + } + + public static explicit operator Grapheme(char ch) => new Grapheme(ch); + + public static explicit operator Grapheme(Rune rune) => new Grapheme(rune.ToString()); + + public static explicit operator Rune(Grapheme grapheme) + { + if (grapheme.str != null && grapheme.str.Length >= 2 && grapheme.str[0] >= 0xD800U && grapheme.str[0] <= 0xDBFFU) + { + return new Rune(grapheme.str[0], grapheme.str[1]); + } + + return new Rune(grapheme.c); + } + + /// + /// Initializes a new instance of the struct with a single character. + /// + /// The character to represent. + public Grapheme(char c) + { + this.c = c; + } + + /// + /// Initializes a new instance of the struct with a string. + /// + /// The string representing the grapheme cluster. + public Grapheme(string str) + { + if (str.Length == 1) + { + // For single characters, we can store them directly in the char field + c = str[0]; + } + else + { + this.str = str; + // For surrogate pairs, use replacement character to avoid storing invalid high surrogate in char field + if (!(str[0] >= 0xD800U && str[0] <= 0xDBFFU)) + c = str[0]; + else + c = (char)0xFFFDU; // Unicode replacement character + } + } + + /// + /// Initializes a new instance of the struct from a ReadOnlySpan of characters. + /// + /// The character span representing the grapheme cluster. + public Grapheme(ReadOnlySpan span) + { + if (span.Length == 1) + { + c = span[0]; + } + else + { + str = span.ToString(); + // For surrogate pairs, use replacement character to avoid storing invalid high surrogate in char field + if (!(str[0] >= 0xD800U && str[0] <= 0xDBFFU)) + c = str[0]; + else + c = (char)0xFFFDU; // Unicode replacement character + } + } + + public static bool operator ==(Grapheme left, Grapheme right) => left.c == right.c && left.str == right.str; + + public static bool operator !=(Grapheme left, Grapheme right) => left.c != right.c || left.str != right.str; + + /// + /// Attempts to remove the last modifier (skin tone, combining mark, etc.) for font fallback. + /// Returns null if no modifier can be removed. + /// + /// + /// + /// var emojiWithSkinTone = new Grapheme("πŸ‘‹πŸ½"); // Waving hand + medium skin tone + /// var baseEmoji = emojiWithSkinTone.RemoveLastModifier(); + /// Console.WriteLine(baseEmoji); // πŸ‘‹ // Waving hand + /// + /// + /// + public Grapheme? RemoveLastModifier() + { + if (str == null) + return null; + + if (Rune.DecodeLastFromUtf16(str, out var _, out int markLength) == OperationStatus.Done) + { + string newStr = str[..^markLength]; + return string.IsNullOrEmpty(newStr) ? null : new Grapheme(newStr); + } + + return null; + } + + public override string ToString() + { + return str ?? c.ToString(); + } + + public bool Equals(Grapheme other) + { + return str == other.str && c == other.c; + } + + public override bool Equals(object? obj) + { + return obj is Grapheme other && this == other; + } + + /// + /// Gets an enumerator that yields instances from the given text. + /// + /// The text to enumerate graphemes from. + /// + /// This method correctly handles complex Unicode scenarios such as: + /// - Emoji with skin tone modifiers (πŸ‘‹πŸ½) + /// - Characters with combining marks (Γ© = e + Β΄) + /// - Regional indicator sequences (country flags) + /// - Zero-width joiner sequences + /// + public static IEnumerable GetGraphemeEnumerator(string text) + { + if (string.IsNullOrEmpty(text)) + yield break; + + int index = 0; + + while (index < text.Length) + { + int length = StringInfo.GetNextTextElementLength(text, index); + + // If the text element is a single character, use the char constructor for efficiency + if (length == 1) + { + yield return new Grapheme(text[index]); + } + else + { + // Otherwise use the string constructor for multi-character elements + yield return new Grapheme(text.Substring(index, length)); + } + + index += length; + } + } + + public override int GetHashCode() + { + return HashCode.Combine(c, str); + } + } +} diff --git a/osu.Framework/Text/ICharacterGlyph.cs b/osu.Framework/Text/ICharacterGlyph.cs index 4860a71122..0260a36c6d 100644 --- a/osu.Framework/Text/ICharacterGlyph.cs +++ b/osu.Framework/Text/ICharacterGlyph.cs @@ -31,7 +31,7 @@ public interface ICharacterGlyph /// /// The character represented by this glyph. /// - char Character { get; } + Grapheme Character { get; } /// /// Retrieves the kerning between this and the one prior to it. diff --git a/osu.Framework/Text/ITexturedCharacterGlyph.cs b/osu.Framework/Text/ITexturedCharacterGlyph.cs index 204a291b21..fb20ceee11 100644 --- a/osu.Framework/Text/ITexturedCharacterGlyph.cs +++ b/osu.Framework/Text/ITexturedCharacterGlyph.cs @@ -24,6 +24,13 @@ public interface ITexturedCharacterGlyph : ICharacterGlyph /// The height of the area that should be drawn. /// float Height { get; } + + /// + /// Whether this character has a coloured texture, typically used for emoji. + /// When true, the character will be rendered using its original texture colours. + /// When false, rendered using the text colour. + /// + bool Coloured { get; } } public static class TexturedCharacterGlyphExtensions @@ -33,6 +40,6 @@ public static class TexturedCharacterGlyphExtensions /// public static bool IsWhiteSpace(this T glyph) where T : ITexturedCharacterGlyph - => char.IsWhiteSpace(glyph.Character); + => glyph.Character.IsWhiteSpace(); } } diff --git a/osu.Framework/Text/ITexturedGlyphLookupStore.cs b/osu.Framework/Text/ITexturedGlyphLookupStore.cs index dee6922664..2d8b258b64 100644 --- a/osu.Framework/Text/ITexturedGlyphLookupStore.cs +++ b/osu.Framework/Text/ITexturedGlyphLookupStore.cs @@ -14,7 +14,7 @@ public interface ITexturedGlyphLookupStore /// This is used to look up a glyph in any font while requiring certain weight / italics specifications. /// The character to retrieve. /// The character glyph. - ITexturedCharacterGlyph? Get(string? fontName, char character); + ITexturedCharacterGlyph? Get(string? fontName, Grapheme character); /// /// Retrieves a glyph from the store asynchronously. @@ -23,6 +23,6 @@ public interface ITexturedGlyphLookupStore /// This is used to look up a glyph in any font while requiring certain weight / italics specifications. /// The character to retrieve. /// The character glyph. - Task GetAsync(string fontName, char character); + Task GetAsync(string fontName, Grapheme character); } } diff --git a/osu.Framework/Text/TextBuilder.cs b/osu.Framework/Text/TextBuilder.cs index 045f43eb52..0d32489931 100644 --- a/osu.Framework/Text/TextBuilder.cs +++ b/osu.Framework/Text/TextBuilder.cs @@ -113,7 +113,7 @@ public virtual void Reset() /// The text to append. public void AddText(string text) { - foreach (char c in text) + foreach (var c in Grapheme.GetGraphemeEnumerator(text)) { if (!AddCharacter(c)) break; @@ -125,7 +125,7 @@ public void AddText(string text) /// /// The character to append. /// Whether characters can still be added. - public bool AddCharacter(char character) + public bool AddCharacter(Grapheme character) { if (!CanAddCharacters) return false; @@ -330,9 +330,9 @@ protected virtual void OnWidthExceeded() private readonly Cached constantWidthCache = new Cached(); - private float getConstantWidth() => constantWidthCache.IsValid ? constantWidthCache.Value : constantWidthCache.Value = getTexturedGlyph(fixedWidthReferenceCharacter)?.Width ?? 0; + private float getConstantWidth() => constantWidthCache.IsValid ? constantWidthCache.Value : constantWidthCache.Value = getTexturedGlyph((Grapheme)fixedWidthReferenceCharacter)?.Width ?? 0; - private bool tryCreateGlyph(char character, out TextBuilderGlyph glyph) + private bool tryCreateGlyph(Grapheme character, out TextBuilderGlyph glyph) { var fontStoreGlyph = getTexturedGlyph(character); @@ -343,7 +343,7 @@ private bool tryCreateGlyph(char character, out TextBuilderGlyph glyph) } // Array.IndexOf is used to avoid LINQ - if (font.FixedWidth && Array.IndexOf(neverFixedWidthCharacters, character) == -1) + if (font.FixedWidth && Array.IndexOf(neverFixedWidthCharacters, character.CharValue) == -1) glyph = new TextBuilderGlyph(fontStoreGlyph, font.Size, getConstantWidth(), useFontSizeAsHeight); else glyph = new TextBuilderGlyph(fontStoreGlyph, font.Size, useFontSizeAsHeight: useFontSizeAsHeight); @@ -351,17 +351,40 @@ private bool tryCreateGlyph(char character, out TextBuilderGlyph glyph) return true; } - private ITexturedCharacterGlyph? getTexturedGlyph(char character) + private ITexturedCharacterGlyph? getTexturedGlyph(Grapheme character) { - return tryGetGlyph(character, font, store) ?? - tryGetGlyph(fallbackCharacter, font, store); + var glyph = tryGetGlyph(character, font, store); + if (glyph != null) + return glyph; - static ITexturedCharacterGlyph? tryGetGlyph(char character, FontUsage font, ITexturedGlyphLookupStore store) + if (tryFallback(character, new Grapheme(fallbackCharacter)) is { } fallback) + return getTexturedGlyph(fallback); + + return null; + + static ITexturedCharacterGlyph? tryGetGlyph(Grapheme character, FontUsage font, ITexturedGlyphLookupStore store) { return store.Get(font.FontName, character) ?? store.Get(font.FontNameNoFamily, character) ?? store.Get(string.Empty, character); } + + static Grapheme? tryFallback(Grapheme character, Grapheme fallbackCharacter) + { + if (character == fallbackCharacter) + { + // If the character is the fallback character, don't try to fallback again + return null; + } + + if (character.RemoveLastModifier() is { } withoutModifier) + { + // If the character has a modifier, remove it and try again + return withoutModifier; + } + + return fallbackCharacter; + } } } } diff --git a/osu.Framework/Text/TextBuilderGlyph.cs b/osu.Framework/Text/TextBuilderGlyph.cs index c1631fef6f..dfb7eae998 100644 --- a/osu.Framework/Text/TextBuilderGlyph.cs +++ b/osu.Framework/Text/TextBuilderGlyph.cs @@ -16,7 +16,8 @@ public struct TextBuilderGlyph : ITexturedCharacterGlyph public readonly float XOffset => ((fixedWidth - Glyph.Width) / 2 ?? Glyph.XOffset) * textSize; public readonly float XAdvance => (fixedWidth ?? Glyph.XAdvance) * textSize; public readonly float Width => Glyph.Width * textSize; - public readonly char Character => Glyph.Character; + public readonly Grapheme Character => Glyph.Character; + public readonly bool Coloured => Glyph.Coloured; public readonly float YOffset { diff --git a/osu.Framework/Text/TexturedCharacterGlyph.cs b/osu.Framework/Text/TexturedCharacterGlyph.cs index 78b7f43a56..4cdbe7608f 100644 --- a/osu.Framework/Text/TexturedCharacterGlyph.cs +++ b/osu.Framework/Text/TexturedCharacterGlyph.cs @@ -9,12 +9,12 @@ namespace osu.Framework.Text public sealed class TexturedCharacterGlyph : ITexturedCharacterGlyph { public Texture Texture { get; } - + public bool Coloured => false; public float XOffset => glyph.XOffset * Scale; public float YOffset => glyph.YOffset * Scale; public float XAdvance => glyph.XAdvance * Scale; public float Baseline => glyph.Baseline * Scale; - public char Character => glyph.Character; + public Grapheme Character => glyph.Character; public float Width => Texture.Width * Scale; public float Height => Texture.Height * Scale;