From 135c8f730cec237bd8eefc48b51230a8ac323e31 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jean-Philippe=20Bruy=C3=A8re?= Date: Fri, 17 Sep 2021 09:04:56 +0000 Subject: [PATCH] crow style tokenizer --- CrowEditBase/src/Compiler/SourceDocument.cs | 12 +- CrowEditBase/src/Compiler/SyntaxAnalyser.cs | 4 +- CrowEditBase/src/Compiler/SyntaxNode.cs | 4 +- CrowEditBase/src/Compiler/Tokenizer.cs | 7 + CrowEditBase/src/CrowEditComponent.cs | 6 +- CrowEditBase/src/LogViewerWidget.cs | 1 + CrowEditBase/src/Plugin.cs | 2 +- CrowEditBase/src/SourceEditor.cs | 60 +++--- plugins/CECrowPlugin/CECrowPlugin.csproj | 2 +- plugins/CECrowPlugin/default.conf | 2 +- plugins/CECrowPlugin/src/CrowService.cs | 69 +++++-- plugins/CECrowPlugin/src/DebugInterface.cs | 13 ++ .../src/StyleParsing/StyleDocument.cs | 66 +++++++ .../src/StyleParsing/StyleTokenType.cs | 36 ++++ .../src/StyleParsing/StyleTokenizer.cs | 183 ++++++++++++++++++ .../src/StyleParsing/SyntaxAnalyser.cs | 43 ++++ .../src/StyleParsing/SyntaxNodes.cs | 31 +++ plugins/CECrowPlugin/ui/winConfiguration.crow | 5 + .../{ImlParsing => Parsing}/SyntaxAnalyser.cs | 22 +-- .../{ImlParsing => Parsing}/SyntaxNodes.cs | 16 +- .../{ImlParsing => Parsing}/XmlDocument.cs | 16 +- .../{ImlParsing => Parsing}/XmlTokenType.cs | 0 .../{ImlParsing => Parsing}/XmlTokenizer.cs | 0 ui/windows/winEditor.crow | 1 + 24 files changed, 516 insertions(+), 85 deletions(-) create mode 100644 plugins/CECrowPlugin/src/StyleParsing/StyleDocument.cs create mode 100644 plugins/CECrowPlugin/src/StyleParsing/StyleTokenType.cs create mode 100644 plugins/CECrowPlugin/src/StyleParsing/StyleTokenizer.cs create mode 100644 plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs create mode 100644 plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs rename plugins/CEXmlPlugin/src/{ImlParsing => Parsing}/SyntaxAnalyser.cs (93%) rename plugins/CEXmlPlugin/src/{ImlParsing => Parsing}/SyntaxNodes.cs (91%) rename plugins/CEXmlPlugin/src/{ImlParsing => Parsing}/XmlDocument.cs (84%) rename plugins/CEXmlPlugin/src/{ImlParsing => Parsing}/XmlTokenType.cs (100%) rename plugins/CEXmlPlugin/src/{ImlParsing => Parsing}/XmlTokenizer.cs (100%) diff --git a/CrowEditBase/src/Compiler/SourceDocument.cs b/CrowEditBase/src/Compiler/SourceDocument.cs index bb4ba3e..f023014 100644 --- a/CrowEditBase/src/Compiler/SourceDocument.cs +++ b/CrowEditBase/src/Compiler/SourceDocument.cs @@ -12,12 +12,12 @@ namespace CrowEditBase { public abstract class SourceDocument : TextDocument { public SourceDocument (string fullPath) - : base (fullPath) { + : base (fullPath) { } protected Token[] tokens; protected SyntaxNode RootNode; protected Token currentToken; - protected SyntaxNode currentNode; + protected SyntaxNode currentNode; public Token[] Tokens => tokens; public Token FindTokenIncludingPosition (int pos) { @@ -56,14 +56,14 @@ namespace CrowEditBase return Colors.DarkGrey; if (tokType.HasFlag (TokenType.Trivia)) return Colors.DimGrey; - if (tokType == TokenType.Keyword) + if (tokType == TokenType.Keyword) return Colors.DarkSlateBlue; - return Colors.Red; + return Colors.Red; } protected abstract Tokenizer CreateTokenizer (); protected abstract SyntaxAnalyser CreateSyntaxAnalyser (); public abstract IList GetSuggestions (int pos); - + /// /// complete current token with selected item from the suggestion overlay. /// It may set a new position or a new selection. @@ -89,7 +89,7 @@ namespace CrowEditBase /*foreach (Token t in Tokens) Console.WriteLine ($"{t,-40} {Source.AsSpan(t.Start, t.Length).ToString()}"); syntaxAnalyser.Root.Dump();*/ - } + } } } \ No newline at end of file diff --git a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs index f58f67d..d3a7f74 100644 --- a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs +++ b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs @@ -8,13 +8,13 @@ using System.Linq; namespace CrowEditBase { public class SyntaxException : Exception { - public readonly Token Token; + public readonly Token Token; public SyntaxException(string message, Token token = default, Exception innerException = null) : base (message, innerException) { Token = token; } } - public abstract class SyntaxAnalyser { + public abstract class SyntaxAnalyser { protected SourceDocument source; public abstract SyntaxNode Root { get; } public List Exceptions { get; protected set; } diff --git a/CrowEditBase/src/Compiler/SyntaxNode.cs b/CrowEditBase/src/Compiler/SyntaxNode.cs index 92b681d..8128d60 100644 --- a/CrowEditBase/src/Compiler/SyntaxNode.cs +++ b/CrowEditBase/src/Compiler/SyntaxNode.cs @@ -10,10 +10,10 @@ namespace CrowEditBase public class SyntaxNode { public SyntaxNode Parent { get; private set; } List children = new List (); - + public readonly Token StartToken; public Token? EndToken { get; set; } - public SyntaxNode (Token tokStart, Token? tokEnd = null) { + public SyntaxNode (Token tokStart, Token? tokEnd = null) { StartToken = tokStart; EndToken = tokEnd; } diff --git a/CrowEditBase/src/Compiler/Tokenizer.cs b/CrowEditBase/src/Compiler/Tokenizer.cs index 80cc2a4..d01640f 100644 --- a/CrowEditBase/src/Compiler/Tokenizer.cs +++ b/CrowEditBase/src/Compiler/Tokenizer.cs @@ -1,9 +1,16 @@ // Copyright (c) 2013-2021 Bruyère Jean-Philippe // // This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) +using System; namespace CrowEditBase { + public class TokenizerException : Exception { + public readonly int Position; + public TokenizerException (int position, string message) : base (message) { + Position = position; + } + } public abstract class Tokenizer { public Tokenizer () {} diff --git a/CrowEditBase/src/CrowEditComponent.cs b/CrowEditBase/src/CrowEditComponent.cs index 9f0147a..d3995a6 100644 --- a/CrowEditBase/src/CrowEditComponent.cs +++ b/CrowEditBase/src/CrowEditComponent.cs @@ -8,7 +8,7 @@ using System.Runtime.CompilerServices; namespace CrowEditBase { - public class CrowEditComponent : IValueChange, ISelectable { + public class CrowEditComponent : IValueChange, ISelectable { #region IValueChange implementation public event EventHandler ValueChanged; public void NotifyValueChanged (string MemberName, object _value) @@ -21,11 +21,11 @@ namespace CrowEditBase NotifyValueChanged (caller, _value); } #endregion - + #region ISelectable implementation bool isSelected; public event EventHandler Selected; - public event EventHandler Unselected; + public event EventHandler Unselected; public virtual bool IsSelected { get => isSelected; diff --git a/CrowEditBase/src/LogViewerWidget.cs b/CrowEditBase/src/LogViewerWidget.cs index 3cab786..e7d1d74 100644 --- a/CrowEditBase/src/LogViewerWidget.cs +++ b/CrowEditBase/src/LogViewerWidget.cs @@ -344,6 +344,7 @@ namespace Crow public override void onMouseLeave(object sender, MouseMoveEventArgs e) { hoverEntryIdx = -1; + RegisterForRedraw (); base.onMouseLeave(sender, e); } public override void onMouseDown(object sender, MouseButtonEventArgs e) diff --git a/CrowEditBase/src/Plugin.cs b/CrowEditBase/src/Plugin.cs index 44356df..e9f0dd1 100644 --- a/CrowEditBase/src/Plugin.cs +++ b/CrowEditBase/src/Plugin.cs @@ -80,7 +80,7 @@ namespace CrowEditBase } } catch (System.Exception ex) { - throw; + Console.WriteLine ($"[Plugin]Error reading 'default.conf' for {FullPath}: {ex.Message}"); } } } diff --git a/CrowEditBase/src/SourceEditor.cs b/CrowEditBase/src/SourceEditor.cs index 37358b6..a20f683 100644 --- a/CrowEditBase/src/SourceEditor.cs +++ b/CrowEditBase/src/SourceEditor.cs @@ -16,7 +16,7 @@ using CrowEditBase; namespace Crow { - public class SourceEditor : Editor { + public class SourceEditor : Editor { object TokenMutex = new object(); @@ -33,7 +33,7 @@ namespace Crow if (suggestions == null || suggestions.Count == 0) hideOverlay (); else - showOverlay (); + showOverlay (); } } bool suggestionsActive => overlay != null && overlay.IsVisible; @@ -44,7 +44,7 @@ namespace Crow return; base.OnTextChanged(sender, e); - + if (!disableSuggestions && HasFocus) tryGetSuggestions (); @@ -53,10 +53,10 @@ namespace Crow //Console.WriteLine ($"{pos}: {suggestionTok.AsString (_text)} {suggestionTok}"); } - + protected void tryGetSuggestions () { - if (currentLoc.HasValue && Document is SourceDocument srcDoc) + if (currentLoc.HasValue && Document is SourceDocument srcDoc) Suggestions = srcDoc.GetSuggestions (lines.GetAbsolutePosition (CurrentLoc.Value)); else Suggestions = null; @@ -67,39 +67,39 @@ namespace Crow overlay = IFace.LoadIMLFragment(@" - + - - + - - + "); overlay.DataSource = this; - overlay.Loaded += (sender, arg) => (sender as ListBox).SelectedIndex = 0; + overlay.Loaded += (sender, arg) => (sender as ListBox).SelectedIndex = 0; } else overlay.IsVisible = true; - overlay.RegisterForLayouting(LayoutingType.Sizing); + overlay.RegisterForLayouting(LayoutingType.Sizing); } } void hideOverlay () { @@ -108,9 +108,9 @@ namespace Crow overlay.IsVisible = false; } void completeToken () { - if (Document is SourceDocument srcDoc) { + if (Document is SourceDocument srcDoc) { TextChange? change = srcDoc.GetCompletionForCurrentToken (overlay.SelectedItem, out TextSpan? nextSelection); - if (change.HasValue) + if (change.HasValue) update (change.Value); if (nextSelection.HasValue) Selection = nextSelection.Value; @@ -121,7 +121,7 @@ namespace Crow hideOverlay (); base.onMouseDown (sender, e); } - + public override void onKeyDown(object sender, KeyEventArgs e) { TextSpan selection = Selection; @@ -162,7 +162,7 @@ namespace Crow disableSuggestions = true; if (IFace.Shift) { - for (int l = lineStart; l <= lineEnd; l++) { + for (int l = lineStart; l <= lineEnd; l++) { if (_text[lines[l].Start] == '\t') update (new TextChange (lines[l].Start, 1, "")); else if (Char.IsWhiteSpace (_text[lines[l].Start])) { @@ -174,10 +174,10 @@ namespace Crow } }else{ - for (int l = lineStart; l <= lineEnd; l++) - update (new TextChange (lines[l].Start, 0, "\t")); + for (int l = lineStart; l <= lineEnd; l++) + update (new TextChange (lines[l].Start, 0, "\t")); } - + selectionStart = new CharLocation (lineStart, 0); CurrentLoc = new CharLocation (lineEnd, lines[lineEnd].Length); @@ -186,7 +186,7 @@ namespace Crow return; } base.onKeyDown(sender, e); - } + } protected override void drawContent (Context gr) { if (!(Document is SourceDocument xmlDoc)) { @@ -200,7 +200,7 @@ namespace Crow base.drawContent (gr); return; } - + Rectangle cb = ClientRectangle; fe = gr.FontExtents; double lineHeight = fe.Ascent + fe.Descent; @@ -262,7 +262,7 @@ namespace Crow TextExtents extents; int tokPtr = 0; Token tok = xmlDoc.Tokens[tokPtr]; - bool multilineToken = false; + bool multilineToken = false; ReadOnlySpan buff = sourceBytes; @@ -272,7 +272,7 @@ namespace Crow if (multilineToken) { if (tok.End < lines[i].End) {//last incomplete line of multiline token - buff = sourceBytes.Slice (lines[i].Start, tok.End - lines[i].Start); + buff = sourceBytes.Slice (lines[i].Start, tok.End - lines[i].Start); } else {//print full line buff = sourceBytes.Slice (lines[i].Start, lines[i].Length); } @@ -301,7 +301,7 @@ namespace Crow gr.MoveTo (pixX, lineHeight * y + fe.Ascent); gr.ShowText (bytes.Slice (0, encodedBytes)); pixX += extents.XAdvance; - x += buff.Length; + x += buff.Length; } if (multilineToken) { @@ -319,7 +319,7 @@ namespace Crow if (HasFocus && selectionNotEmpty) { RectangleD lineRect = new RectangleD (cb.X, lineHeight * y + cb.Top, pixX, lineHeight); RectangleD selRect = lineRect; - + if (i >= selStart.Line && i <= selEnd.Line) { if (selStart.Line == selEnd.Line) { selRect.X = selStart.VisualCharXPosition + cb.X; @@ -365,7 +365,7 @@ namespace Crow x = 0; pixX = 0; - + y++; @@ -379,13 +379,13 @@ namespace Crow continue; } else if (tok2.Type == TokenType.WhiteSpace) { x += tok2.Length; - pixX += spacePixelWidth * tok2.Length;*/ - } + pixX += spacePixelWidth * tok2.Length;*/ + } //gr.Translate (ScrollX, ScrollY); } finally { xmlDoc.ExitReadLock (); } - } + } } } \ No newline at end of file diff --git a/plugins/CECrowPlugin/CECrowPlugin.csproj b/plugins/CECrowPlugin/CECrowPlugin.csproj index f982122..1a634b7 100644 --- a/plugins/CECrowPlugin/CECrowPlugin.csproj +++ b/plugins/CECrowPlugin/CECrowPlugin.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 false - + diff --git a/plugins/CECrowPlugin/default.conf b/plugins/CECrowPlugin/default.conf index 69586b6..4226396 100644 --- a/plugins/CECrowPlugin/default.conf +++ b/plugins/CECrowPlugin/default.conf @@ -1 +1 @@ -FileAssociations=CECrowPlugin.ImlDocument:.crow,.iml,.itmp,.template \ No newline at end of file +FileAssociations=CECrowPlugin.ImlDocument:.crow,.iml,.itmp,.template,.tmp;CECrowPlugin.StyleDocument:.style \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/CrowService.cs b/plugins/CECrowPlugin/src/CrowService.cs index 71bacd4..86ff6a4 100644 --- a/plugins/CECrowPlugin/src/CrowService.cs +++ b/plugins/CECrowPlugin/src/CrowService.cs @@ -24,6 +24,8 @@ namespace Crow public class CrowService : Service { public CrowService () : base () { + loadCrowAssemblies (); + initCommands (); //resolve other plugins dependencies @@ -152,15 +154,6 @@ namespace Crow NotifyValueChanged ("PreviewHasError", PreviewHasError); } } - public string CrowDbgAssemblyLocation { - get => Configuration.Global.Get ("CrowDbgAssemblyLocation"); - set { - if (CrowDbgAssemblyLocation == value) - return; - Configuration.Global.Set ("CrowDbgAssemblyLocation", value); - NotifyValueChanged(value); - } - } public bool DebugLogIsEnabled { get => debugLogIsEnabled; set { @@ -235,6 +228,20 @@ namespace Crow x = mouseScreenPos.X; y = mouseScreenPos.Y; } + void saveCrowAssemblies () { + if (crowAssemblies.Count > 0) + Configuration.Global.Set ("CrowAssemblies", crowAssemblies.Aggregate ((a, b)=> $"{a};{b}")); + else + Configuration.Global.Set ("CrowAssemblies", ""); + } + void loadCrowAssemblies () { + crowAssemblies.Clear (); + if (!Configuration.Global.TryGet ("CrowAssemblies", out string assemblies)) + return; + foreach (string a in assemblies.Split (';')) + crowAssemblies.Add (a); + } + public override void Start() { if (CurrentState == Status.Running) @@ -246,13 +253,21 @@ namespace Crow updateCrowDebuggerState($"Crow.dll for debugging file not found"); return; } + List additionalResolvePath = new List(); + additionalResolvePath.Add (System.IO.Path.GetDirectoryName(CrowDbgAssemblyLocation)); + foreach (string assemblyPath in crowAssemblies) + additionalResolvePath.Add (System.IO.Path.GetDirectoryName(assemblyPath)); crowLoadCtx = new AssemblyLoadContext("CrowDebuggerLoadContext"); crowLoadCtx.ResolvingUnmanagedDll += resolveUnmanaged; crowLoadCtx.Resolving += (context, assemblyName) => { - return crowLoadCtx.LoadFromAssemblyPath ( - System.IO.Path.Combine ( - System.IO.Path.GetDirectoryName(CrowDbgAssemblyLocation), assemblyName.Name + ".dll")); + foreach (string path in additionalResolvePath) { + string assemblyPath = System.IO.Path.Combine (path, assemblyName.Name + ".dll"); + if (!File.Exists (assemblyPath)) + continue; + return crowLoadCtx.LoadFromAssemblyPath (assemblyPath); + } + return null; }; //crowLoadCtx.Resolving += (ctx,name) => AssemblyLoadContext.Default.LoadFromAssemblyName (name); @@ -260,6 +275,9 @@ namespace Crow crowAssembly = crowLoadCtx.LoadFromAssemblyPath (CrowDbgAssemblyLocation); thisAssembly = crowLoadCtx.LoadFromAssemblyPath (new Uri(Assembly.GetExecutingAssembly().CodeBase).LocalPath); + foreach (string assemblyPath in crowAssemblies) + crowLoadCtx.LoadFromAssemblyPath (assemblyPath); + Type debuggerType = crowAssembly.GetType("Crow.DbgLogger"); DebugLogIsEnabled = (bool)debuggerType.GetField("IsEnabled").GetValue(null); @@ -330,6 +348,20 @@ namespace Crow CurrentState = Status.Paused; } public override string ConfigurationWindowPath => "#CECrowPlugin.ui.winConfiguration.crow"; + + public string CrowDbgAssemblyLocation { + get => Configuration.Global.Get ("CrowDbgAssemblyLocation"); + set { + if (CrowDbgAssemblyLocation == value) + return; + Configuration.Global.Set ("CrowDbgAssemblyLocation", value); + NotifyValueChanged(value); + } + } + //assemblies with crow resources in order of loading + IList crowAssemblies = new ObservableList (); + public IList CrowAssemblies => crowAssemblies; + public ActionCommand CMDOptions_SelectCrowAssemblyLocation => new ActionCommand ("...", () => { FileDialog dlg = App.LoadIMLFragment (@" @@ -339,6 +371,19 @@ namespace Crow dlg.DataSource = this; } ); + public ActionCommand CMDOptions_AddCrowAssembly => new ActionCommand ("Add Assembly with Crow Ressource", + () => { + FileDialog dlg = App.LoadIMLFragment (@" + "); + dlg.OkClicked += (sender, e) => { + crowAssemblies.Add ((sender as FileDialog).SelectedFileFullPath); + saveCrowAssemblies (); + }; + dlg.DataSource = this; + } + ); + protected override void onStateChange(Status previousState, Status newState) { diff --git a/plugins/CECrowPlugin/src/DebugInterface.cs b/plugins/CECrowPlugin/src/DebugInterface.cs index dd2d49c..9d35681 100644 --- a/plugins/CECrowPlugin/src/DebugInterface.cs +++ b/plugins/CECrowPlugin/src/DebugInterface.cs @@ -185,5 +185,18 @@ namespace CECrowPlugin return result; return base.GetStreamFromPath (path); } + public override Type GetWidgetTypeFromName (string typeName){ + if (knownCrowWidgetTypes.ContainsKey (typeName)) + return knownCrowWidgetTypes [typeName]; + foreach (Assembly a in System.Runtime.Loader.AssemblyLoadContext.GetLoadContext (Assembly.GetExecutingAssembly ()).Assemblies) { + foreach (Type expT in a.GetExportedTypes ()) { + if (expT.Name != typeName) + continue; + knownCrowWidgetTypes.Add (typeName, expT); + return expT; + } + } + return null; + } } } \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/StyleParsing/StyleDocument.cs b/plugins/CECrowPlugin/src/StyleParsing/StyleDocument.cs new file mode 100644 index 0000000..9e3692b --- /dev/null +++ b/plugins/CECrowPlugin/src/StyleParsing/StyleDocument.cs @@ -0,0 +1,66 @@ +// Copyright (c) 2013-2021 Bruyère Jean-Philippe +// +// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) + +using System; +using System.Linq; +using Crow.Text; +using System.Collections.Generic; +using System.Diagnostics; +using Crow; +using IML = Crow.IML; +using System.Collections; +using System.Reflection; +using CrowEditBase; +using static CrowEditBase.CrowEditBase; + +using CERoslynPlugin; + +namespace CECrowPlugin +{ + public class StyleDocument : SourceDocument { + + + public StyleDocument (string fullPath) : base (fullPath) { + App.GetService ()?.Start (); + + /*if (project is MSBuildProject msbp) { + if (msbp.IsCrowProject) + }*/ + } + + protected override Tokenizer CreateTokenizer() => new StyleTokenizer (); + protected override SyntaxAnalyser CreateSyntaxAnalyser() => new StyleSyntaxAnalyser (this); + + public override IList GetSuggestions (int pos) { + currentToken = FindTokenIncludingPosition (pos); + currentNode = FindNodeIncludingPosition (pos); + return null; + } + public override TextChange? GetCompletionForCurrentToken (object suggestion, out TextSpan? newSelection) { + newSelection = null; + return null; + } + public override Color GetColorForToken(TokenType tokType) + { + StyleTokenType xmlTokType = (StyleTokenType)tokType; + if (xmlTokType.HasFlag (StyleTokenType.Punctuation)) + return Colors.DarkGrey; + if (xmlTokType.HasFlag (StyleTokenType.Trivia)) + return Colors.DimGrey; + if (xmlTokType == StyleTokenType.MemberName) + return Colors.Blue; + if (xmlTokType == StyleTokenType.ConstantName) + return Colors.DarkCyan; + else if (xmlTokType.HasFlag (StyleTokenType.Name)) + return Colors.Green; + if (xmlTokType == StyleTokenType.MemberValuePart) + return Colors.OrangeRed; + if (xmlTokType == StyleTokenType.EqualSign) + return Colors.Black; + if (xmlTokType == StyleTokenType.Unknown) + return Colors.Red; + return Colors.YellowGreen; + } + } +} \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/StyleParsing/StyleTokenType.cs b/plugins/CECrowPlugin/src/StyleParsing/StyleTokenType.cs new file mode 100644 index 0000000..e6609f5 --- /dev/null +++ b/plugins/CECrowPlugin/src/StyleParsing/StyleTokenType.cs @@ -0,0 +1,36 @@ +// Copyright (c) 2013-2021 Bruyère Jean-Philippe +// +// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) + +using System; + +namespace CECrowPlugin +{ + [Flags] + public enum StyleTokenType { + Unknown, + Trivia = 0x0100, + WhiteSpace = 0x4100, + Tabulation = 0x4101, + LineBreak = 0x4102, + LineCommentStart = 0x0102, + LineComment = 0x0103, + BlockCommentStart = 0x0104, + BlockComment = 0x0105, + BlockCommentEnd = 0x0106, + Name = 0x0200, + StyleKey = 0x0201,//may be a class name or a style name. + MemberName = 0x0202, + ConstantName = 0x0203, + Punctuation = 0x0400, + OpeningBrace = 0x0401,// '{' + ClosingBrace = 0x0402,// '}' + Comma = 0x0403,// ',' + EndOfExpression = 0x0404,// ';' + EqualSign = 0x0801, + MemberValuePart = 0x2000, + MemberValueOpen = 0x2401, + MemberValueClose = 0x2402, + ConstantRefOpen = 0x2403,// '${' + } +} \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/StyleParsing/StyleTokenizer.cs b/plugins/CECrowPlugin/src/StyleParsing/StyleTokenizer.cs new file mode 100644 index 0000000..f9e9c62 --- /dev/null +++ b/plugins/CECrowPlugin/src/StyleParsing/StyleTokenizer.cs @@ -0,0 +1,183 @@ +// Copyright (c) 2013-2021 Bruyère Jean-Philippe +// +// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) + +using System; +using Crow.Text; +using System.Collections.Generic; +using CrowEditBase; +using System.Globalization; +using Crow.Coding; + +namespace CECrowPlugin { + public class StyleTokenizer : Tokenizer { + enum States { + classNames, members, value, endOfStatement + } + + States curState; + int startOfTok; + + + public StyleTokenizer () {} + + bool readName (ref SpanCharReader reader) { + if (reader.EndOfSpan) + return false; + char c = reader.Peak; + if (char.IsLetter(c) || c == '_' ) { + reader.Advance (); + while (reader.TryPeak (ref c)) { + if (!char.IsLetterOrDigit(c)) { + UnicodeCategory uc = Char.GetUnicodeCategory (c); + if (uc != UnicodeCategory.NonSpacingMark && + uc != UnicodeCategory.SpacingCombiningMark && + uc != UnicodeCategory.ConnectorPunctuation && + uc != UnicodeCategory.Format) + return true; + } + reader.Advance (); + } + return true; + } + return false; + } + protected List Toks; + + void skipWhiteSpaces (ref SpanCharReader reader) { + while(!reader.EndOfSpan) { + switch (reader.Peak) { + case '\x85': + case '\x2028': + case '\xA': + reader.Read(); + addTok (ref reader, StyleTokenType.LineBreak); + break; + case '\xD': + reader.Read(); + if (reader.IsNextCharIn ('\xA', '\x85')) + reader.Read(); + addTok (ref reader, StyleTokenType.LineBreak); + break; + case '\x20': + case '\x9': + char c = reader.Read(); + while (reader.TryPeak (c)) + reader.Read(); + addTok (ref reader, c == '\x20' ? StyleTokenType.WhiteSpace : StyleTokenType.Tabulation); + break; + default: + return; + } + } + } + void addTok (ref SpanCharReader reader, Enum tokType) { + if (reader.CurrentPosition == startOfTok) + return; + Toks.Add (new Token((TokenType)tokType, startOfTok, reader.CurrentPosition)); + startOfTok = reader.CurrentPosition; + } + public override Token[] Tokenize (string source) { + SpanCharReader reader = new SpanCharReader(source); + + startOfTok = 0; + int curObjectLevel = 0; + curState = States.classNames; + Toks = new List(100); + + while(!reader.EndOfSpan) { + + skipWhiteSpaces (ref reader); + + if (reader.EndOfSpan) + break; + + switch (reader.Peak) { + case '/': + reader.Advance (); + if (reader.TryPeak ('/')) { + reader.Advance (); + addTok (ref reader, StyleTokenType.LineCommentStart); + reader.AdvanceUntilEol (); + addTok (ref reader, StyleTokenType.LineComment); + } else if (reader.TryPeak ('*')) { + reader.Advance (); + addTok (ref reader, StyleTokenType.BlockCommentStart); + if (reader.TryReadUntil ("*/")) { + addTok (ref reader, StyleTokenType.BlockComment); + reader.Advance (2); + addTok (ref reader, StyleTokenType.BlockCommentEnd); + } + } + break; + case ',': + reader.Advance (); + addTok (ref reader, StyleTokenType.Comma); + curState = States.classNames; + break; + case '{': + reader.Advance (); + addTok (ref reader, StyleTokenType.OpeningBrace); + curState = States.members; + break; + case '}': + reader.Advance (); + addTok (ref reader, StyleTokenType.ClosingBrace); + curState = States.classNames; + break; + case '=': + reader.Advance (); + addTok (ref reader, StyleTokenType.EqualSign); + curState = States.value; + break; + case '"': + reader.Advance (); + addTok (ref reader, StyleTokenType.MemberValueOpen); + + while (!reader.EndOfSpan) { + if (reader.TryPeak ("${")) { + addTok (ref reader, StyleTokenType.MemberValuePart); + reader.Advance (2); + addTok (ref reader, StyleTokenType.ConstantRefOpen); + + while (!reader.EndOfSpan) { + if (reader.TryPeak ('}')) { + addTok (ref reader, StyleTokenType.ConstantName); + reader.Read (); + addTok (ref reader, StyleTokenType.ClosingBrace); + break; + } + reader.Advance (); + } + continue; + } else if (reader.TryPeak ('\"')) { + addTok (ref reader, StyleTokenType.MemberValuePart); + reader.Advance (); + addTok (ref reader, StyleTokenType.MemberValueClose); + break; + } + reader.Advance (); + } + curState = States.endOfStatement; + break; + case ';': + reader.Advance(); + addTok (ref reader, StyleTokenType.EndOfExpression); + curState = States.members; + break; + default: + if (readName (ref reader)) { + addTok (ref reader, StyleTokenType.Name); + break; + } + reader.Advance (); + addTok (ref reader, StyleTokenType.Unknown); + break; + } + + } + + return Toks.ToArray(); + } + } +} diff --git a/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs b/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs new file mode 100644 index 0000000..a9ebfbe --- /dev/null +++ b/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs @@ -0,0 +1,43 @@ +// Copyright (c) 2021-2021 Bruyère Jean-Philippe +// +// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) +using System; +using System.Collections.Generic; +using System.Linq; +using CrowEditBase; + +namespace CECrowPlugin +{ + public class StyleSyntaxAnalyser : SyntaxAnalyser { + public override SyntaxNode Root => CurrentNode; + public StyleSyntaxAnalyser (StyleDocument source) : base (source) { + this.source = source; + } + + SyntaxNode CurrentNode; + Token previousTok; + IEnumerator iter; + + public override void Process () { + StyleDocument doc = source as StyleDocument; + Exceptions = new List (); + CurrentNode = new StyleRootSyntax (doc); + previousTok = default; + iter = doc.Tokens.AsEnumerable().GetEnumerator (); + + bool notEndOfSource = iter.MoveNext (); + while (notEndOfSource) { + if (!iter.Current.Type.HasFlag (TokenType.Trivia)) { + } + + previousTok = iter.Current; + notEndOfSource = iter.MoveNext (); + } + while (CurrentNode.Parent != null) { + if (!CurrentNode.EndToken.HasValue) + CurrentNode.EndToken = previousTok; + CurrentNode = CurrentNode.Parent; + } + } + } +} \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs b/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs new file mode 100644 index 0000000..cf32f72 --- /dev/null +++ b/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs @@ -0,0 +1,31 @@ +// Copyright (c) 2013-2021 Bruyère Jean-Philippe +// +// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT) +using System; +using System.Collections.Generic; +using System.Linq; + +using CrowEditBase; + +namespace CECrowPlugin +{ + + public class StyleRootSyntax : SyntaxNode { + internal readonly StyleDocument source; + public override SyntaxNode Root => this; + public StyleRootSyntax (StyleDocument source) + : base (source.Tokens.FirstOrDefault (), source.Tokens.LastOrDefault ()) { + this.source = source; + } + } + + public class AttributeSyntax : SyntaxNode { + public Token? NameToken { get; internal set; } + public Token? EqualToken { get; internal set; } + public Token? ValueOpenToken { get; internal set; } + public Token? ValueCloseToken { get; internal set; } + public Token? ValueToken { get; internal set; } + public AttributeSyntax (Token startTok) : base (startTok) {} + public override bool IsComplete => base.IsComplete & NameToken.HasValue & EqualToken.HasValue & ValueToken.HasValue & ValueOpenToken.HasValue & ValueCloseToken.HasValue; + } +} \ No newline at end of file diff --git a/plugins/CECrowPlugin/ui/winConfiguration.crow b/plugins/CECrowPlugin/ui/winConfiguration.crow index 67dbf12..8ad2c70 100644 --- a/plugins/CECrowPlugin/ui/winConfiguration.crow +++ b/plugins/CECrowPlugin/ui/winConfiguration.crow @@ -20,6 +20,11 @@