From: Jean-Philippe Bruyère Date: Sat, 30 Jan 2021 06:13:38 +0000 (+0100) Subject: custom utf8 encoding with tabulation handling, linebreak detection X-Git-Tag: v0.9.5-beta~89 X-Git-Url: https://git.osiis.dedyn.io/?a=commitdiff_plain;h=a6373abaaae947ea87a8f037a5fb81ba32f311a6;p=jp%2Fcrow.git custom utf8 encoding with tabulation handling, linebreak detection --- diff --git a/Crow/src/ExtensionsMethods.cs b/Crow/src/ExtensionsMethods.cs index 4961ea90..f8aefaf1 100644 --- a/Crow/src/ExtensionsMethods.cs +++ b/Crow/src/ExtensionsMethods.cs @@ -197,29 +197,6 @@ namespace Crow internal static bool IsAnyLineBreakCharacter (this char c) => c == '\n' || c == '\r' || c == '\u0085' || c == '\u2028' || c == '\u2029'; - internal static ReadOnlySpan GetLine (this string str, TextLine ls) { - if (ls.Start >= str.Length) - return "".AsSpan (); - return str.AsSpan ().Slice (ls.Start, ls.Length); - } - internal static ReadOnlySpan GetLine (this string str, TextLine ls, int offset) { - int start = ls.Start + offset; - if (start >= str.Length) - return "".AsSpan (); - return str.AsSpan ().Slice (start, ls.Length); - - } - internal static ReadOnlySpan GetLineIncludingLineBreak (this string str, TextLine ls) { - if (ls.Start >= str.Length) - return "".AsSpan (); - return str.AsSpan ().Slice (ls.Start, ls.LengthIncludingLineBreak); - } - internal static ReadOnlySpan GetLineIncludingLineBreak (this string str, TextLine ls, int offset) { - int start = ls.Start + offset; - if (start >= str.Length) - return "".AsSpan (); - return str.AsSpan ().Slice (start, ls.LengthIncludingLineBreak); - } } } diff --git a/Crow/src/Mono.Cairo/Context.cs b/Crow/src/Mono.Cairo/Context.cs index a8dfac34..45600929 100644 --- a/Crow/src/Mono.Cairo/Context.cs +++ b/Crow/src/Mono.Cairo/Context.cs @@ -882,7 +882,7 @@ namespace Crow.Cairo { public void ShowText (ReadOnlySpan s, int tabSize) { int size = s.Length * 4 + 1; Span bytes = size > 512 ? new byte[size] : stackalloc byte[size]; - int encodedBytes = Encoding.UTF8.GetBytes(s, bytes); + int encodedBytes = Crow.Text.Encoding.ToUtf8 (s, bytes, tabSize); bytes[encodedBytes] = 0; NativeMethods.cairo_show_text (handle, ref bytes.Slice (0, encodedBytes + 1).GetPinnableReference ()); @@ -891,7 +891,7 @@ namespace Crow.Cairo { TextExtents extents; int size = s.Length * 4 + 1; Span bytes = size > 512 ? new byte[size] : stackalloc byte[size]; - int encodedBytes = Encoding.UTF8.GetBytes (s, bytes); + int encodedBytes = Crow.Text.Encoding.ToUtf8 (s, bytes, tabSize); bytes[encodedBytes] = 0; NativeMethods.cairo_text_extents (handle, ref bytes.Slice (0, encodedBytes + 1).GetPinnableReference (), out extents); @@ -900,7 +900,7 @@ namespace Crow.Cairo { public void TextExtents (ReadOnlySpan s, int tabSize, out TextExtents extents) { int size = s.Length * 4 + 1; Span bytes = size > 512 ? new byte[size] : stackalloc byte[size]; - int encodedBytes = Encoding.UTF8.GetBytes (s, bytes); + int encodedBytes = Crow.Text.Encoding.ToUtf8 (s, bytes, tabSize); bytes[encodedBytes] = 0; NativeMethods.cairo_text_extents (handle, ref bytes.Slice (0, encodedBytes + 1).GetPinnableReference (), out extents); @@ -908,7 +908,7 @@ namespace Crow.Cairo { public void TextExtents (ReadOnlySpan s, out TextExtents extents) { int size = s.Length * 4 + 1; Span bytes = size > 512 ? new byte[size] : stackalloc byte[size]; - int encodedBytes = Encoding.UTF8.GetBytes (s, bytes); + int encodedBytes = Crow.Text.Encoding.ToUtf8 (s, bytes); bytes[encodedBytes] = 0; NativeMethods.cairo_text_extents (handle, ref bytes.Slice (0, encodedBytes + 1).GetPinnableReference (), out extents); diff --git a/Crow/src/Text/Encoding.cs b/Crow/src/Text/Encoding.cs new file mode 100644 index 00000000..f89ae0e4 --- /dev/null +++ b/Crow/src/Text/Encoding.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Crow.Text +{ + public class Encoding + { + public static int ToUtf8 (ReadOnlySpan source, Span buff, int tabWidth = 4) { + int c = 0; + int encodedBytes = 0; + int encodedChar = 0; + while (c < source.Length) { + if (source[c] < 0xD800) { + if (source[c] == '\t') { + int encTabWidth = tabWidth - encodedChar % tabWidth; + for (int i = 0; i < encTabWidth; i++) { + buff[encodedBytes++] = 0x20; + encodedChar++; + } + c++; + continue; + } + /*unsafe { + fixed (void* bp = &source.GetPinnableReference()) + fixed (byte* buffer = &buff.GetPinnableReference()) + { + return Utf16toUtf8 ((byte*)bp, buffer, tabWidth); + + } + }*/ + + + if (source[c] < 0x80) { //1 byte + buff[encodedBytes++] = (byte)source[c++]; + }else if (source[c] < 0x7FF) { //2 bytes + buff[encodedBytes++] = (byte)((source[c] >> 6) + 0xC0); + buff[encodedBytes++] = (byte)((source[c] & 0x3F) + 0x80); + c++; + } else { //3 bytes + //ushort ch = (ushort)(source[c] - 0x10000u); + buff[encodedBytes++] = (byte)((source[c] >> 12) + 0xE0); + buff[encodedBytes++] = (byte)((source[c] >> 6) & 0x00BF); + buff[encodedBytes++] = (byte)((source[c] & 0x3F) + 0x80); + c++; + } + + encodedChar++; + continue; + } + throw new NotImplementedException(); + } + return encodedBytes; + } + + + /*unsafe static int Utf16toUtf8 (byte* cptr, byte* buff, int tabWidth) { + + }*/ + } +} diff --git a/Crow/src/Text/Extensions.cs b/Crow/src/Text/Extensions.cs new file mode 100644 index 00000000..7f5a53b7 --- /dev/null +++ b/Crow/src/Text/Extensions.cs @@ -0,0 +1,50 @@ +using System; + +namespace Crow.Text +{ + public static class Extensions + { + public static ReadOnlySpan GetLine (this string str, TextLine ls) { + if (ls.Start >= str.Length) + return "".AsSpan (); + return str.AsSpan ().Slice (ls.Start, ls.Length); + } + public static ReadOnlySpan GetLine (this string str, TextLine ls, int offset) { + int start = ls.Start + offset; + if (start >= str.Length) + return "".AsSpan (); + return str.AsSpan ().Slice (start, ls.Length); + + } + public static ReadOnlySpan GetLineIncludingLineBreak (this string str, TextLine ls) { + if (ls.Start >= str.Length) + return "".AsSpan (); + return str.AsSpan ().Slice (ls.Start, ls.LengthIncludingLineBreak); + } + public static ReadOnlySpan GetLineBreak (this string str, TextLine ls) { + if (ls.LineBreakLength == 0) + return "".AsSpan (); + return str.AsSpan ().Slice (ls.End, ls.LineBreakLength); + } + public static ReadOnlySpan GetLineIncludingLineBreak (this string str, TextLine ls, int offset) { + int start = ls.Start + offset; + if (start >= str.Length) + return "".AsSpan (); + return str.AsSpan ().Slice (start, ls.LengthIncludingLineBreak); + } + + public static ReadOnlySpan ToCharSpan (this LineBreakKind lineBreak) { + switch (lineBreak) { + case LineBreakKind.Unix: + return "\n".AsSpan (); + case LineBreakKind.Windows: + return "\r\n".AsSpan (); + case LineBreakKind.Other: + return "\r".AsSpan (); + default: + return "\r\n".AsSpan (); + } + } + + } +} diff --git a/Crow/src/Text/TextLine.cs b/Crow/src/Text/TextLine.cs index 53190748..f5798a31 100644 --- a/Crow/src/Text/TextLine.cs +++ b/Crow/src/Text/TextLine.cs @@ -5,6 +5,13 @@ using System.Text; namespace Crow.Text { + public enum LineBreakKind + { + Undefined, + Unix, + Windows, + Other + } [DebuggerDisplay ("{Start}, {Length}, {LengthInPixel}")] public struct TextLine : IComparable { @@ -30,20 +37,8 @@ namespace Crow.Text } public TextLine WithStartOffset (int start) => new TextLine (Start + start, End, EndIncludingLineBreak); + + public int CompareTo (TextLine other) => Start - other.Start; - /*public ReadOnlySpan GetSubString (string str) { - if (Start >= str.Length) - return "".AsSpan(); - return str.Length - Start < Length ? - str.AsSpan().Slice (Start, Length) : - str.AsSpan().Slice (Start); -} -public ReadOnlySpan GetSubStringIncludingLineBreak (string str) { - if (Start >= str.Length) - return "".AsSpan (); - return (str.Length - Start < LengthIncludingLineBreak) ? - str.AsSpan ().Slice (Start, LengthIncludingLineBreak) : - str.AsSpan ().Slice (Start); -}*/ } } diff --git a/Crow/src/Text/TextLineCollection.cs b/Crow/src/Text/TextLineCollection.cs index a505409f..586c0d8c 100644 --- a/Crow/src/Text/TextLineCollection.cs +++ b/Crow/src/Text/TextLineCollection.cs @@ -26,7 +26,7 @@ namespace Crow.Text } #endregion - public int GetAbsolutePosition (CharLocation loc) => lines[loc.Line].Start + loc.Column; + public int GetAbsolutePosition (CharLocation loc) => lines[loc.Line].Start + loc.Column; public CharLocation GetLocation (int absolutePosition) { TextLine tl = new TextLine (absolutePosition); int result = lines.AsSpan (0, length).BinarySearch (tl); @@ -43,12 +43,13 @@ namespace Crow.Text } public int Count => length; public bool IsReadOnly => false; + public bool IsEmpty => length == 0; public TextLine this[int index] { get => lines[index]; set => lines[index] = value; } public void Add (TextLine item) { if (lines.Length < length + 1) { - TextLine[] tmp = new TextLine[length + 4]; + TextLine[] tmp = new TextLine[length * 2]; lines.AsSpan ().CopyTo (tmp); lines = tmp; } diff --git a/Crow/src/Widgets/Label.cs b/Crow/src/Widgets/Label.cs index 81632de7..45ea16bf 100644 --- a/Crow/src/Widgets/Label.cs +++ b/Crow/src/Widgets/Label.cs @@ -139,6 +139,7 @@ namespace Crow { if (value == _multiline) return; _multiline = value; + lines = new LineCollection (_multiline ? 4 : 1); NotifyValueChangedAuto (_multiline); RegisterForGraphicUpdate(); } @@ -446,52 +447,78 @@ namespace Crow { */ protected bool textMeasureIsUpToDate = false; Size cachedTextSize = default(Size); - protected LineCollection lines; + protected LineCollection lines; + protected string LineBreak = null; + bool mixedLineBreak = false; + + protected void detectLineBreak () { + if (!_multiline) + return; + mixedLineBreak = false; + + if (lines.Count == 0 || lines[0].LineBreakLength == 0) { + LineBreak = Environment.NewLine; + return; + } + LineBreak = _text.GetLineBreak (lines[0]).ToString (); + + for (int i = 1; i < lines.Count; i++) { + ReadOnlySpan lb = _text.GetLineBreak (lines[i]); + if (!lb.SequenceEqual (LineBreak)) { + mixedLineBreak = true; + break; + } + } + } + protected void getLines () { + + if (lines == null) + lines = new LineCollection (_multiline ? 4 : 1); + else + lines.Clear (); + if (string.IsNullOrEmpty (_text)) { - lines = new LineCollection(new TextLine[] { new TextLine (0, 0, 0) }); + lines.Add (new TextLine (0, 0, 0)); return; } if (!_multiline) { - lines = new LineCollection (new TextLine[] { new TextLine (0, _text.Length, _text.Length) }); + lines.Add (new TextLine (0, _text.Length, _text.Length)); return; } - - List _lines = new List (); + int start = 0, i = 0; while (i < _text.Length) { char c = _text[i]; if (c == '\r') { if (++i < _text.Length) { if (_text[i] == '\n') - _lines.Add (new TextLine (start, i - 1, ++i)); + lines.Add (new TextLine (start, i - 1, ++i)); else - _lines.Add (new TextLine (start, i - 1, i)); + lines.Add (new TextLine (start, i - 1, i)); } else - _lines.Add (new TextLine (start, i - 1, i)); + lines.Add (new TextLine (start, i - 1, i)); start = i; } else if (c == '\n') { if (++i < _text.Length) { if (_text[i] == '\r') - _lines.Add (new TextLine (start, i - 1, ++i)); + lines.Add (new TextLine (start, i - 1, ++i)); else - _lines.Add (new TextLine (start, i - 1, i)); + lines.Add (new TextLine (start, i - 1, i)); } else - _lines.Add (new TextLine (start, i - 1, i)); + lines.Add (new TextLine (start, i - 1, i)); start = i; } else if (c == '\u0085' || c == '\u2028' || c == '\u2029') - _lines.Add (new TextLine (start, i - 1, i)); + lines.Add (new TextLine (start, i - 1, i)); else i++; } if (start < i) - _lines.Add (new TextLine (start, _text.Length, _text.Length)); + lines.Add (new TextLine (start, _text.Length, _text.Length)); else - _lines.Add (new TextLine (_text.Length, _text.Length, _text.Length)); - - lines = new LineCollection (_lines.ToArray ()); + lines.Add (new TextLine (_text.Length, _text.Length, _text.Length)); } void resetLocationXs () { if (currentLoc.HasValue) { @@ -540,7 +567,7 @@ namespace Crow { #region GraphicObject overrides public override int measureRawSize(LayoutingType lt) { - if (lines == null) + if ((bool)lines?.IsEmpty) getLines (); if (!textMeasureIsUpToDate) { @@ -646,7 +673,7 @@ namespace Crow { if (bytes.Length < size) bytes = size > 512 ? new byte[size] : stackalloc byte[size]; - encodedBytes = Encoding.UTF8.GetBytes (_text.GetLine (lines[i]), bytes); + encodedBytes = Crow.Text.Encoding.ToUtf8 (_text.GetLine (lines[i]), bytes); bytes[encodedBytes++] = 0; if (lines[i].LengthInPixel < 0) { @@ -868,6 +895,7 @@ namespace Crow { double cPos = Width.IsFit && !Multiline ? 0 : getX (clientWidth, ls); if (loc.Column >= 0) { + //int encodedBytes = Crow.Text.Encoding2.ToUtf8 (curLine.Slice (0, loc.Column), bytes); loc.VisualCharXPosition = gr.TextExtents (curLine.Slice (0, loc.Column), Interface.TAB_SIZE).XAdvance + cPos; location = loc; return; @@ -878,7 +906,7 @@ namespace Crow { for (int i = 0; i < ls.Length; i++) { - int encodedBytes = Encoding.UTF8.GetBytes (curLine.Slice (i, 1), bytes); + int encodedBytes = Crow.Text.Encoding.ToUtf8 (curLine.Slice (i, 1), bytes); bytes[encodedBytes] = 0; gr.TextExtents (bytes, out te); diff --git a/Crow/src/Widgets/TextBox.cs b/Crow/src/Widgets/TextBox.cs index 61af4127..282e01c0 100644 --- a/Crow/src/Widgets/TextBox.cs +++ b/Crow/src/Widgets/TextBox.cs @@ -32,18 +32,25 @@ namespace Crow TextSpan selection = Selection; switch (key) { case Key.Backspace: - if (selection.Length == 0) { + if (selection.IsEmpty) { if (selection.Start == 0) return; - update (new TextChange (selection.Start - 1, 1, "")); + if (currentLoc.Value.Column == 0) { + int lbLength = lines[currentLoc.Value.Line - 1].LineBreakLength; + update (new TextChange (selection.Start - lbLength, lbLength, "")); + }else + update (new TextChange (selection.Start - 1, 1, "")); } else update (new TextChange (selection.Start, selection.Length, "")); break; case Key.Delete: - if (selection.Length == 0) { + if (selection.IsEmpty) { if (selection.Start == Text.Length) return; - update (new TextChange (selection.Start, 1, "")); + if (currentLoc.Value.Column >= lines[currentLoc.Value.Line].Length) + update (new TextChange (selection.Start, lines[currentLoc.Value.Line].LineBreakLength, "")); + else + update (new TextChange (selection.Start, 1, "")); } else { if (IFace.Shift) IFace.Clipboard = Text.AsSpan (selection.Start, selection.End).ToString (); @@ -58,10 +65,12 @@ namespace Crow break; case Key.KeypadEnter: case Key.Enter: - if (Multiline) - update (new TextChange (selection.Start, selection.Length, "\n")); - else - OnValidate (this, new ValidateEventArgs(_text)); + if (Multiline) { + if (string.IsNullOrEmpty (LineBreak)) + detectLineBreak (); + update (new TextChange (selection.Start, selection.Length, LineBreak)); + } else + OnValidate (this, new ValidateEventArgs (_text)); break; case Key.Escape: selectionStart = null; @@ -102,14 +111,13 @@ namespace Crow _text = tmp.ToString (); getLines (); + selectionStart = null; + currentLoc = lines.GetLocation (change.Start + change.ChangedText.Length); textMeasureIsUpToDate = false; NotifyValueChanged ("Text", Text); OnTextChanged (this, new TextChangeEventArgs (change)); - selectionStart = null; - currentLoc = lines.GetLocation (change.Start + change.ChangedText.Length); - RegisterForGraphicUpdate (); } }