diff --git a/PSReadLine/Changes.txt b/PSReadLine/Changes.txt index 0b99add8..060f481d 100644 --- a/PSReadLine/Changes.txt +++ b/PSReadLine/Changes.txt @@ -1,3 +1,9 @@ +### Unreleased + +- Correct cursor columns for Tamil nonspacing marks while keeping spacing vowel signs one cell wide. + Preserve attached marks when truncating text, mapping cursor positions, and counting wrapped lines. + This is a scoped Tamil correction, not general Unicode grapheme-cluster support. + ### [2.4.5] - 2025-10-22 - Replace `DOTNET_SKIP_FIRST_TIME_EXPERIENCE` with `DOTNET_NOLOGO` (#4916) (Thanks @xtqqczze!) diff --git a/PSReadLine/Render.Helper.cs b/PSReadLine/Render.Helper.cs index 8fc56bea..00ed8b29 100644 --- a/PSReadLine/Render.Helper.cs +++ b/PSReadLine/Render.Helper.cs @@ -99,6 +99,13 @@ internal static int LengthInBufferCells(char c) return Char.IsControl(c) ? 2 : 1; } + // Tamil anusvara, vowel sign II, and pulli are nonspacing marks. + // Spacing vowel signs still contribute a cell in Terminal's graphemes mode. + if (c is '\u0b82' or '\u0bc0' or '\u0bcd') + { + return 0; + } + // The following is based on http://www.cl.cam.ac.uk/~mgk25/c/wcwidth.c // which is derived from http://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt @@ -125,6 +132,15 @@ private static string SubstringByCells(string text, int countOfCells) return SubstringByCells(text, 0, countOfCells); } + private static int SkipZeroWidthCharacters(StringBuilder text, int start, int end) + { + while (start < end && LengthInBufferCells(text[start]) == 0) + { + start++; + } + return start; + } + private static string SubstringByCells(string text, int start, int countOfCells) { int length = SubstringLengthByCells(text, start, countOfCells); @@ -138,6 +154,11 @@ private static int SubstringLengthByCells(string text, int countOfCells) private static int SubstringLengthByCells(string text, int start, int countOfCells) { + if (countOfCells <= 0) + { + return 0; + } + int cellLength = 0; int charLength = 0; @@ -151,11 +172,6 @@ private static int SubstringLengthByCells(string text, int start, int countOfCel } charLength++; - - if (cellLength == countOfCells) - { - return charLength; - } } return charLength; @@ -168,19 +184,34 @@ private static int SubstringLengthByCellsFromEnd(string text, int countOfCells) private static int SubstringLengthByCellsFromEnd(string text, int start, int countOfCells) { + if (countOfCells <= 0) + { + return 0; + } + int cellLength = 0; int charLength = 0; + int pendingMarks = 0; for (int i = start; i >= 0; i--) { - cellLength += LengthInBufferCells(text[i]); + int width = LengthInBufferCells(text[i]); + if (width == 0) + { + pendingMarks++; + continue; + } + + cellLength += width; if (cellLength > countOfCells) { return charLength; } - charLength++; + // Include trailing marks only when their base fits too. + charLength += pendingMarks + 1; + pendingMarks = 0; if (cellLength == countOfCells) { @@ -188,7 +219,7 @@ private static int SubstringLengthByCellsFromEnd(string text, int start, int cou } } - return charLength; + return charLength + pendingMarks; } private static (int newStart, int newEnd) TrimSubstringInPlace(string text, int start, int end) diff --git a/PSReadLine/Render.cs b/PSReadLine/Render.cs index 253f7fbe..2ef5bb5f 100644 --- a/PSReadLine/Render.cs +++ b/PSReadLine/Render.cs @@ -75,6 +75,11 @@ public int PhysicalLineCount(int bufferWidth, int initialX, out int lenLastPhysi } int size = PSConsoleReadLine.LengthInBufferCells(c); + if (size == 0) + { + continue; + } + if (x == 0 && lenLastPhysicalLine > 0) { y++; @@ -1395,7 +1400,8 @@ internal Point ConvertOffsetToPoint(int offset, StringBuilder buffer = null) // If cursor is at column 0 and the next character is newline, let the next loop // iteration increment y. - if (x != 0 || !(i + 1 < offset && buffer[i + 1] == '\n')) + int next = SkipZeroWidthCharacters(buffer, i + 1, offset); + if (x != 0 || !(next < offset && buffer[next] == '\n')) { y += 1; } @@ -1435,7 +1441,7 @@ private int ConvertLineAndColumnToOffset(Point point) // the correct column if (point.Y == y && point.X <= x) { - return offset; + return SkipZeroWidthCharacters(_buffer, offset, _buffer.Length); } char c = _buffer[offset]; if (c == '\n') @@ -1461,7 +1467,8 @@ private int ConvertLineAndColumnToOffset(Point point) // If cursor is at column 0 and the next character is newline, let the next loop // iteration increment y. - if (x != 0 || !(offset + 1 < _buffer.Length && _buffer[offset + 1] == '\n')) + int next = SkipZeroWidthCharacters(_buffer, offset + 1, _buffer.Length); + if (x != 0 || !(next < _buffer.Length && _buffer[next] == '\n')) { y += 1; } @@ -1752,7 +1759,28 @@ internal RenderDataOffset ConvertPointToRenderDataOffset(int initialX, int initi } else if (point.X == x) { - // 'x' is pointing to where the next visible character would be rendered. + // Include attached marks before returning the offset + // at this column, including across color sequences. + int next = i + 1; + while (next < line.Length) + { + if (line[next] == 0x1b && next + 1 < line.Length && line[next + 1] == '[') + { + next += 2; + while (next < line.Length && line[next] != 'm') + { + next++; + } + next++; + continue; + } + if (LengthInBufferCells(line[next]) != 0) + { + break; + } + visibleCharIndex++; + next++; + } return new RenderDataOffset(logicalLineIndex, visibleCharIndex + 1); } } diff --git a/test/CellWidthTest.cs b/test/CellWidthTest.cs new file mode 100644 index 00000000..47a919f6 --- /dev/null +++ b/test/CellWidthTest.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the 2-Clause BSD License. + +using System; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using Microsoft.PowerShell; +using Xunit; + +namespace Test +{ + public class CellWidthTest + { + static CellWidthTest() + { + RuntimeHelpers.RunClassConstructor(typeof(ReadLine).TypeHandle); + } + + [Theory] + [InlineData("", 0)] + [InlineData("ABC", 3)] + [InlineData("\u0003", 2)] + [InlineData("\u754c", 2)] + [InlineData("\u2764\ufe0f", 2)] + [InlineData("\u0b95", 1)] + [InlineData("\u0b95\u0b82", 1)] + [InlineData("\u0b95\u0bcd", 1)] + [InlineData("\u0b95\u0bc0", 1)] + [InlineData("\u0b95\u0bbe", 2)] + [InlineData("\u0b95\u0bbf", 2)] + [InlineData("\u0b95\u0bc1", 2)] + [InlineData("\u001b[31m\u0b95\u0bcd\u001b[0m", 1)] + [InlineData("\u0bb5\u0ba3\u0b95\u0bcd\u0b95\u0bae\u0bcd \u0b87\u0baa\u0bcd\u0baa\u0b9f\u0bbf \u0b87\u0bb0\u0bc1\u0b95\u0bcd\u0b95\u0bc0\u0b99\u0bcd\u0b95 ?", 21)] + public void TamilCellWidths(string text, int expected) + { + Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(text)); + string padded = "x" + text + "y"; + Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(padded, 1, padded.Length - 1)); + Assert.Equal(expected, PSConsoleReadLine.LengthInBufferCells(new StringBuilder(padded), 1, padded.Length - 1)); + } + + [Theory] + [InlineData("\u0b95\u0bcdX", 0, 1, false, 2)] + [InlineData("\u0b95\u0bc0\u0b82X", 0, 1, false, 3)] + [InlineData("X\u0b95\u0bcdY", 1, 1, false, 2)] + [InlineData("\u0b95\u0bcd", 0, 0, false, 0)] + [InlineData("\u0b95\u0bcd", 1, 0, true, 0)] + [InlineData("\u0b95\u0bcd", 1, 1, true, 2)] + [InlineData("X\u0b95\u0bcdY", 2, 1, true, 2)] + [InlineData("\u754c\u0bcd", 1, 1, true, 0)] + [InlineData("\u754c\u0bcdX", 2, 2, true, 1)] + [InlineData("ABC", 0, 2, false, 2)] + [InlineData("ABC", 2, 2, true, 2)] + public void CellSubstringKeepsAttachedMarks(string text, int start, int cells, bool fromEnd, int expected) + { + string name = fromEnd ? "SubstringLengthByCellsFromEnd" : "SubstringLengthByCells"; + var method = typeof(PSConsoleReadLine).GetMethod( + name, BindingFlags.Static | BindingFlags.NonPublic, null, + new[] { typeof(string), typeof(int), typeof(int) }, null); + Assert.NotNull(method); + Assert.Equal(expected, (int)method.Invoke(null, new object[] { text, start, cells })); + } + + [Theory] + [InlineData("abc\u0b95\u0bcd", 1, 4)] + [InlineData("abc\u0b95\u0bcdX", 2, 1)] + [InlineData("abc\u0b95\u001b[31m\u0bcd\u001b[0m", 1, 4)] + public void AttachedMarkDoesNotWrapToAnotherPhysicalLine(string text, int lines, int lastLineWidth) + { + var data = new RenderedLineData(text, isFirstLogicalLine: true); + Assert.Equal(lines, data.PhysicalLineCount(4, 0, out int actualLastLineWidth)); + Assert.Equal(lastLineWidth, actualLastLineWidth); + } + } +} diff --git a/test/MockConsole.cs b/test/MockConsole.cs index 99069867..ac6799c5 100644 --- a/test/MockConsole.cs +++ b/test/MockConsole.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; using Microsoft.PowerShell.Internal; @@ -200,6 +201,12 @@ public virtual void SetWindowPosition(int left, int top) { } + protected static bool IsTamilNonspacingMark(char c) + { + return c >= '\u0b80' && c <= '\u0bff' + && char.GetUnicodeCategory(c) == UnicodeCategory.NonSpacingMark; + } + public virtual void SetCursorPosition(int left, int top) { if (left != CursorLeft || top != CursorTop) @@ -225,6 +232,11 @@ public virtual void Write(string s) var writePos = CursorTop * BufferWidth + CursorLeft; for (int i = 0; i < s.Length; i++) { + if (IsTamilNonspacingMark(s[i])) + { + continue; + } + if (s[i] == (char) 0x1b) { // Escape sequence - limited support here, and assumed to be well formed. @@ -454,6 +466,11 @@ public override void Write(string s) var writePos = (_offset + CursorTop) * BufferWidth + CursorLeft; for (int i = 0; i < s.Length; i++) { + if (IsTamilNonspacingMark(s[i])) + { + continue; + } + if (s[i] == (char)0x1b) { // Escape sequence - limited support here, and assumed to be well formed. diff --git a/test/RenderTest.cs b/test/RenderTest.cs index cc72f367..e0b26cf5 100644 --- a/test/RenderTest.cs +++ b/test/RenderTest.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Management.Automation; using System.Text; using Microsoft.PowerShell; @@ -8,6 +9,65 @@ namespace Test { public partial class ReadLine { + [SkippableFact] + public void TamilNonspacingCursorPosition() + { + TestSetup(KeyMode.Cmd); + const string text = "\u0bb5\u0ba3\u0b95\u0bcd\u0b95\u0bae\u0bcd \u0b87\u0baa\u0bcd\u0baa\u0b9f\u0bbf \u0b87\u0bb0\u0bc1\u0b95\u0bcd\u0b95\u0bc0\u0b99\u0bcd\u0b95 ?"; + Test(text, Keys( + text.Select(c => new ConsoleKeyInfo(c, 0, false, false, false)), + CheckThat(() => AssertCursorLeftIs(25)), + _.Home, + CheckThat(() => AssertCursorLeftIs(4)), + _.End, + CheckThat(() => AssertCursorLeftIs(25))), prompt: "PS> "); + } + + [SkippableFact] + public void TamilNonspacingCoordinateMappings() + { + TestSetup(KeyMode.Cmd); + const string text = "\u0b95\u0bcdX"; + Test(text, Keys(text.Select(c => new ConsoleKeyInfo(c, 0, false, false, false)), CheckThat(() => + { + var instance = (PSConsoleReadLine)_psrlInstance; + var map = typeof(PSConsoleReadLine).GetMethod( + "ConvertLineAndColumnToOffset", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + Assert.Equal(2, (int)map.Invoke(instance, new object[] { new Point { X = 1, Y = 0 } })); + + foreach (string line in new[] { text, "\u001b[31m\u0b95\u001b[0m\u0bcdX" }) + { + var data = new RenderData + { + bufferWidth = 60, + cursorLeft = 1, + cursorTop = 0, + lines = new[] { new RenderedLineData(line, isFirstLogicalLine: true) } + }; + var offset = instance.ConvertPointToRenderDataOffset(0, 0, data); + Assert.Equal(0, offset.LogicalLineIndex); + Assert.Equal(2, offset.VisibleCharIndex); + var point = instance.ConvertRenderDataOffsetToPoint(0, 0, 60, data, offset); + Assert.Equal(1, point.X); + Assert.Equal(0, point.Y); + } + }))); + } + + [SkippableFact] + public void TamilNonspacingMarkBeforeWrappedNewline() + { + TestSetup(KeyMode.Cmd); + Test("X", Keys('X', CheckThat(() => + { + var text = new StringBuilder(new string('a', _console.BufferWidth - 1) + "\u0b95\u0bcd\nX"); + var point = ((PSConsoleReadLine)_psrlInstance).ConvertOffsetToPoint(text.Length, text); + Assert.Equal(ScreenReaderModeEnabled ? 1 : ContinuationPromptLength + 1, point.X); + Assert.Equal(1, point.Y); + }))); + } + [SkippableFact] public void ClearScreen() {