From: Jean-Philippe Bruyère Date: Sat, 8 Mar 2025 16:03:49 +0000 (+0100) Subject: syntax analysis revamping wip X-Git-Url: https://git.osiis.dedyn.io/?a=commitdiff_plain;h=b34733376a11b72225d61e0c5f49753e8c5e4d71;p=jp%2Fcrowedit.git syntax analysis revamping wip --- diff --git a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs index 4097874..449fc0a 100644 --- a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs +++ b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs @@ -11,12 +11,10 @@ namespace CrowEditBase { public abstract class SyntaxAnalyser { protected SourceDocument document; - protected LineCollection lines; protected SyntaxRootNode Root; - public IEnumerable Exceptions => Root?.GetAllExceptions(); + public IEnumerable Exceptions => null;// Root?.GetAllExceptions(); public SyntaxAnalyser (SourceDocument document) { this.document = document; - this.lines = document.Lines; } public abstract Task Process (); @@ -81,7 +79,7 @@ namespace CrowEditBase /// /// The final token of this node /// the endline number of this node - protected void finishCurrentNode (int endTokenOffsetFromCurrentTokIdx = 0) { + /*protected void finishCurrentNode (int endTokenOffsetFromCurrentTokIdx = 0) { int lastTokOffset = tokIdx - currentNode.TokenIndexBase + endTokenOffsetFromCurrentTokIdx; currentNode.lastTokenOfset = lastTokOffset < 0 ? null : lastTokOffset; if (endTokenOffsetFromCurrentTokIdx < 0) { @@ -96,7 +94,7 @@ namespace CrowEditBase currentNode = currentNode.Parent; } protected void setCurrentNodeEndLine (int endLine) - => currentNode.EndLine = endLine; + => currentNode.EndLine = endLine;*/ protected bool skipTrivia(bool skipLineBreaks = true) { while (tryPeekFlag(out Token tok, TokenType.Trivia)) { if (tok.Type == TokenType.LineBreak) { @@ -109,8 +107,8 @@ namespace CrowEditBase return !EOF; } protected void addException(string message) { - CharLocation loc = lines.GetLocation(curTok.Start); - currentNode.AddException(new SyntaxException(message, loc, curTok)); + /*CharLocation loc = lines.GetLocation(curTok.Start); + currentNode.AddException(new SyntaxException(message, loc, curTok));*/ } diff --git a/CrowEditBase/src/Compiler/SyntaxNode.cs b/CrowEditBase/src/Compiler/SyntaxNode.cs index e886313..d720aae 100644 --- a/CrowEditBase/src/Compiler/SyntaxNode.cs +++ b/CrowEditBase/src/Compiler/SyntaxNode.cs @@ -9,41 +9,36 @@ using Crow; namespace CrowEditBase { - /*public class SingleTokenSyntax { + public class SingleTokenSyntax : SyntaxNode { Token token; - - }*/ - public class SyntaxNode : CrowEditComponent { - #region CTOR - internal SyntaxNode () {} - public SyntaxNode (int startLine, int tokenBase, int? lastTokenIdx = null) { - StartLine = startLine; - TokenIndexBase = tokenBase; - if (lastTokenIdx.HasValue) - lastTokenOfset = lastTokenIdx - tokenBase; - } - #endregion - + public override TokenType Type => token.Type; + public override int SpanStart => token.Start; + public override int SpanEnd => token.End; + public SingleTokenSyntax(Token tok) { + token = tok; + } + } + public class MultiNodeSyntax : SyntaxNode { internal List children = new List (); + public IEnumerable Children => children; + public SyntaxNode AddChild (SyntaxNode child) { + children.Add (child); + child.Parent = this; + return child; + } + public void RemoveChild (SyntaxNode child) { + children.Remove (child); + child.Parent = null; + } + public IEnumerable GetChilds () => children.OfType(); + + public override bool HasChilds => children.Count > 0; + public override int SpanStart => HasChilds ? children[0].SpanStart : 0; + public override int SpanEnd => HasChilds ? children[children.Count - 1].SpanEnd : 0; - internal int? lastTokenOfset; - internal int lineCount; - - #region Folding and ?expand? - bool _isExpanded; internal bool isFolded; - - public bool isExpanded { - get => _isExpanded; - set { - if (_isExpanded == value) - return; - _isExpanded = value; - NotifyValueChanged (_isExpanded); - } - } - public virtual bool IsFoldable => IsComplete && !(Parent != Root && Parent.StartLine == StartLine) && lineCount > 1; + public virtual bool IsFoldable => IsComplete && !(Parent != Root && Parent.StartLocation.Line == StartLocation.Line) && LineCount > 1; public void ExpandToTheTop () { isExpanded = true; Parent?.ExpandToTheTop (); @@ -52,14 +47,14 @@ namespace CrowEditBase isFolded = false; Parent.UnfoldToTheTop (); } - public IEnumerable VisibleFoldableNodes { + public IEnumerable VisibleFoldableNodes { get { if (IsFoldable) { yield return this; } if (!isFolded) { - foreach (SyntaxNode n in Children) { - foreach (SyntaxNode folds in n.VisibleFoldableNodes) + foreach (MultiNodeSyntax n in Children.OfType()) { + foreach (MultiNodeSyntax folds in n.VisibleFoldableNodes) yield return folds; } } @@ -68,32 +63,63 @@ namespace CrowEditBase public virtual int FoldedLineCount { get { if (isFolded) - return lineCount; + return LineCount; int tmp = 0; if (HasChilds) { - foreach (SyntaxNode n in children.Where (c => c.IsFoldable)) + foreach (MultiNodeSyntax n in children.OfType().Where (c => c.IsFoldable)) tmp += n.FoldedLineCount; } return tmp; } } + public override SyntaxNode FindNodeIncludingSpan (TextSpan span) { + foreach (SyntaxNode node in children) { + if (node.Contains (span)) + return node.FindNodeIncludingSpan (span); + } + return this; + } + public override SyntaxNode FindNodeIncludingPosition (int pos) { + foreach (SyntaxNode node in children) { + if (node.Contains (pos)) + return node.FindNodeIncludingPosition (pos); + } + return this; + } + } + public abstract class SyntaxNode : CrowEditComponent { + + + #region Folding and ?expand? + bool _isExpanded; + public bool isExpanded { + get => _isExpanded; + set { + if (_isExpanded == value) + return; + _isExpanded = value; + NotifyValueChanged (_isExpanded); + } + } #endregion + public MultiNodeSyntax Parent { get; internal set; } + public virtual SyntaxRootNode Root => Parent.Root; + public virtual TokenType Type => TokenType.Unknown; + public virtual int SpanStart => 0; + public virtual int SpanEnd => 0; - public IEnumerable Children => children; - public bool HasChilds => children.Count > 0; - public SyntaxNode Parent { get; private set; } - public virtual bool IsComplete => lastTokenOfset.HasValue; + public virtual bool HasChilds => false; + public virtual bool IsComplete => false; - public virtual SyntaxRootNode Root => Parent.Root; - public int StartLine { get; private set; } - public virtual int LineCount => lineCount; + /*public int StartLine { get; private set; } + public virtual int LineCount => lineCount;*/ - List exceptions = new List(); + /*List exceptions = new List(); public IEnumerable Exceptions => exceptions; public void AddException(SyntaxException e) => exceptions.Add(e); public void ResetExceptions(SyntaxException e) => exceptions.Clear(); @@ -104,7 +130,7 @@ namespace CrowEditBase foreach (SyntaxException ce in n.GetAllExceptions()) yield return ce; } - } + }*/ protected Token getTokenByIndex (int idx) => Root.GetTokenByIndex(idx); @@ -134,7 +160,7 @@ namespace CrowEditBase public virtual SyntaxNode NextSiblingOrParentsNextSibling => NextSibling ?? Parent.NextSiblingOrParentsNextSibling; - public virtual int TokenIndexBase { get; private set; } + /*public virtual int TokenIndexBase { get; private set; } public virtual int TokenCount => lastTokenOfset.HasValue ? lastTokenOfset.Value + 1 : 0; public int? LastTokenIndex => lastTokenOfset.HasValue ? TokenIndexBase + lastTokenOfset.Value : null; @@ -143,26 +169,12 @@ namespace CrowEditBase lineCount = value - StartLine + 1; } get => StartLine + lineCount - 1; - } - public TextSpan Span { - get { - Token startTok = getTokenByIndex(TokenIndexBase); - Token endTok = LastTokenIndex.HasValue ? getTokenByIndex (LastTokenIndex.Value) : startTok; - return new TextSpan (startTok.Start, endTok.End); - } - } - public int SpanStart; + }*/ + public TextSpan Span => new TextSpan (SpanStart, SpanEnd); + public CharLocation StartLocation => Root.GetLocation(SpanStart); + public CharLocation EndLocation => Root.GetLocation(SpanEnd); + public int LineCount => EndLocation.Line - StartLocation.Line + 1; - public SyntaxNode AddChild (SyntaxNode child) { - children.Add (child); - child.Parent = this; - return child; - } - public void RemoveChild (SyntaxNode child) { - children.Remove (child); - child.Parent = null; - } - public IEnumerable GetChilds () => children.OfType(); /* public void Replace (SyntaxNode newNode) { Parent.replaceChild (this, newNode); @@ -188,20 +200,14 @@ namespace CrowEditBase curNode = curNode.Parent; } }*/ - void offset (int tokenOffset, int lineOffset) { + /*void offset (int tokenOffset, int lineOffset) { TokenIndexBase += tokenOffset; StartLine += lineOffset; foreach (SyntaxNode child in children) { child.offset (tokenOffset, lineOffset); } - } - public SyntaxNode FindNodeIncludingPosition (int pos) { - foreach (SyntaxNode node in children) { - if (node.Contains (pos)) - return node.FindNodeIncludingPosition (pos); - } - return this; - } + }*/ +/* public T FindNodeIncludingPosition (int pos) { foreach (SyntaxNode node in children) { if (node.Contains (pos)) @@ -209,21 +215,17 @@ namespace CrowEditBase } return this is T tt ? tt : default; - } - public SyntaxNode FindNodeIncludingSpan (TextSpan span) { - foreach (SyntaxNode node in children) { - if (node.Contains (span)) - return node.FindNodeIncludingSpan (span); - } - return this; - } + }*/ + public virtual SyntaxNode FindNodeIncludingPosition (int pos) => this; + public virtual SyntaxNode FindNodeIncludingSpan (TextSpan span) => this; + public bool Contains (int pos) => Span.Contains (pos); public bool Contains (TextSpan span) => Span.Contains (span); - public void Dump (int level = 0) { + /*public void Dump (int level = 0) { Console.WriteLine ($"{new string('\t', level)}{this}"); foreach (SyntaxNode node in children) node.Dump (level + 1); - } + }*/ //public override string ToString() => $"l:({StartLine,3},{LineCount,3}) tks:{TokenIndexBase},{TokenCount} {this.GetType().Name}"; public override string ToString() => $"{this.GetType().Name}"; public string AsText() { @@ -234,7 +236,7 @@ namespace CrowEditBase public class CompareOnStartLine : IComparer { - public int Compare(SyntaxNode x, SyntaxNode y) => x.StartLine - y.StartLine; + public int Compare(SyntaxNode x, SyntaxNode y) => x.StartLocation.Line - y.StartLocation.Line; } } } \ No newline at end of file diff --git a/CrowEditBase/src/Compiler/SyntaxRootNode.cs b/CrowEditBase/src/Compiler/SyntaxRootNode.cs index 20f683c..44f7492 100644 --- a/CrowEditBase/src/Compiler/SyntaxRootNode.cs +++ b/CrowEditBase/src/Compiler/SyntaxRootNode.cs @@ -6,21 +6,23 @@ using Crow.Text; namespace CrowEditBase { - public abstract class SyntaxRootNode : SyntaxNode { - public SyntaxRootNode (ReadOnlyMemory source, Token[] tokens) { - this.source = source; + public abstract class SyntaxRootNode : MultiNodeSyntax { + public SyntaxRootNode (ReadOnlyTextBuffer buffer, Token[] tokens) { + this.buffer = buffer; this.tokens = tokens; } - protected readonly ReadOnlyMemory source; + protected readonly ReadOnlyTextBuffer buffer; protected Token[] tokens; - public override int TokenIndexBase => 0; - public override int TokenCount => tokens == null ? 0 : Math.Max (0, tokens.Length - 1); public override SyntaxRootNode Root => this; + + public override bool IsFoldable => false; - public override SyntaxNode NextSiblingOrParentsNextSibling => null; public override void UnfoldToTheTop() {} + public override SyntaxNode NextSiblingOrParentsNextSibling => null; public ReadOnlySpan Tokens => tokens; + + public string GetTokenStringByIndex (int idx) => tokens != null ? idx >= 0 && idx < tokens.Length ? GetText(tokens[idx].Span).ToString() : null : null; public Token GetTokenByIndex (int idx) => tokens != null ? @@ -32,7 +34,9 @@ namespace CrowEditBase return idx == 0 ? 0 : idx < 0 ? ~idx - 1 : idx; } public ReadOnlySpan GetText(TextSpan span) => - source.Span.Slice(span.Start, span.Length); + buffer.Source.Span.Slice(span.Start, span.Length); + public CharLocation GetLocation(int pos) => buffer.Lines.GetLocation(pos); + public string GetTokenString(Token tok) => GetText(tok.Span).ToString(); } diff --git a/CrowEditBase/src/Compiler/TokenType.cs b/CrowEditBase/src/Compiler/TokenType.cs index e35993c..d34c355 100644 --- a/CrowEditBase/src/Compiler/TokenType.cs +++ b/CrowEditBase/src/Compiler/TokenType.cs @@ -19,7 +19,17 @@ namespace CrowEditBase BlockCommentEnd = 0x0106, Name = 0x0200, Punctuation = 0x0400, + OpenParen = 0x0401, + CloseParen = 0x0402, + OpenBracket = 0x0403, + CloseBracket = 0x0404, + OpenBrace = 0x0405, + CloseBrace = 0x0406, + DoubleQuote = 0x0407, + SingleQuote = 0x0408, + Operator = 0x0800, Keyword = 0x1000, + Syntax = 0x8000 } } \ No newline at end of file diff --git a/CrowEditBase/src/SourceEditor.cs b/CrowEditBase/src/SourceEditor.cs index ce40fb6..9a1ba91 100644 --- a/CrowEditBase/src/SourceEditor.cs +++ b/CrowEditBase/src/SourceEditor.cs @@ -170,8 +170,8 @@ namespace CrowEditBase return; currentLoc = value; if (currentLoc.HasValue) { - SyntaxNode fold = getFoldContainingLine (currentLoc.Value.Line); - while (fold != null && fold.StartLine == currentLoc.Value.Line) + MultiNodeSyntax fold = getFoldContainingLine (currentLoc.Value.Line); + while (fold != null && fold.StartLocation.Line == currentLoc.Value.Line) fold = fold.Parent; fold?.UnfoldToTheTop(); updateCurrentTokAndNode(); @@ -208,7 +208,7 @@ namespace CrowEditBase hideOverlay (); if (mouseIsInMargin) { if (e.Button == MouseButton.Left && mouseIsInFoldRect) { - SyntaxNode curNode = getFoldStartingAt (hoverLoc.Value.Line); + MultiNodeSyntax curNode = getFoldStartingAt (hoverLoc.Value.Line); if (curNode != null) { curNode.isFolded = !curNode.isFolded; textMeasureIsUpToDate = false; @@ -346,9 +346,9 @@ namespace CrowEditBase } if (Document is SourceDocument doc) { switch (e.Key) { - case Key.F3: + /*case Key.F3: doc.Root?.Dump(); - break; + break;*/ case Key.Enter: case Key.KeypadEnter: //doc.updateCurrentTokAndNode (Selection.Start); @@ -385,23 +385,23 @@ namespace CrowEditBase } #endregion - SyntaxNode getFoldStartingAt (int line) { + MultiNodeSyntax getFoldStartingAt (int line) { if (!(Document is SourceDocument doc)) return null; - IEnumerable folds = doc.Root.VisibleFoldableNodes; + IEnumerable folds = doc.Root.VisibleFoldableNodes; if (folds == null) return null; - return folds.FirstOrDefault (n => n.StartLine == line); + return folds.FirstOrDefault (n => n.StartLocation.Line == line); } - SyntaxNode getFoldContainingLine (int line) { + MultiNodeSyntax getFoldContainingLine (int line) { if (!(Document is SourceDocument doc)) return null; doc.EnterReadLock(); try { - IEnumerable folds = doc.Root.VisibleFoldableNodes; + IEnumerable folds = doc.Root.VisibleFoldableNodes; if (folds == null) return null; - return folds.LastOrDefault (n => n.StartLine <= line && n.EndLine >= line); + return folds.LastOrDefault (n => n.StartLocation.Line <= line && n.EndLocation.Line >= line); } finally { doc.ExitReadLock (); } @@ -415,16 +415,16 @@ namespace CrowEditBase int foldedLines = 0; if (!doc.IsParsed) return 0; - IEnumerator foldsEnum = doc.Root.VisibleFoldableNodes.GetEnumerator(); + IEnumerator foldsEnum = doc.Root.VisibleFoldableNodes.GetEnumerator(); bool notEndOfFolds = foldsEnum.MoveNext(); - while (notEndOfFolds && foldsEnum.Current.StartLine < absoluteLine) { + while (notEndOfFolds && foldsEnum.Current.StartLocation.Line < absoluteLine) { if (foldsEnum.Current.isFolded) { foldedLines += foldsEnum.Current.LineCount - 1; SyntaxNode nextNode = foldsEnum.Current.NextSiblingOrParentsNextSibling; if (nextNode == null) break; notEndOfFolds = foldsEnum.MoveNext(); - while (notEndOfFolds && foldsEnum.Current.StartLine < nextNode.StartLine) + while (notEndOfFolds && foldsEnum.Current.StartLocation.Line < nextNode.StartLocation.Line) notEndOfFolds = foldsEnum.MoveNext(); } else notEndOfFolds = foldsEnum.MoveNext(); @@ -442,20 +442,20 @@ namespace CrowEditBase int foldedLines = 0; if (!doc.IsParsed) return 0; - IEnumerator nodeEnum = doc.Root.VisibleFoldableNodes.GetEnumerator (); + IEnumerator nodeEnum = doc.Root.VisibleFoldableNodes.GetEnumerator (); if (!nodeEnum.MoveNext()) return 0; int l = 0; while (l < visualLine + foldedLines) { - if (nodeEnum.Current.StartLine == l) { + if (nodeEnum.Current.StartLocation.Line == l) { if (nodeEnum.Current.isFolded) { - foldedLines += nodeEnum.Current.lineCount - 1; + foldedLines += nodeEnum.Current.LineCount - 1; SyntaxNode nextNode = nodeEnum.Current.NextSiblingOrParentsNextSibling; if (nextNode == null || !nodeEnum.MoveNext()) return foldedLines; - while (nodeEnum.Current.StartLine < nextNode.StartLine) { + while (nodeEnum.Current.StartLocation.Line < nextNode.StartLocation.Line) { if (!nodeEnum.MoveNext()) return foldedLines; } @@ -651,10 +651,10 @@ namespace CrowEditBase int linesToSkip = (int)Math.Floor(ScrollY / lineHeight); - IEnumerator foldsEnum = doc.Root.VisibleFoldableNodes.GetEnumerator (); + IEnumerator foldsEnum = doc.Root.VisibleFoldableNodes.GetEnumerator (); bool notEndOfFolds = foldsEnum.MoveNext(); - while (notEndOfFolds && foldsEnum.Current.StartLine < linesToSkip) { + while (notEndOfFolds && foldsEnum.Current.StartLocation.Line < linesToSkip) { if (foldsEnum.Current.isFolded) linesToSkip += foldsEnum.Current.LineCount-1; notEndOfFolds = foldsEnum.MoveNext(); @@ -663,14 +663,14 @@ namespace CrowEditBase int curLine = linesToSkip; pixY = -(ScrollY % lineHeight); - IEnumerator exceptionLines = doc.Root.GetAllExceptions()?.Select(e=>e.Location.Line).Order().GetEnumerator(); - bool hasExceptions = exceptionLines.MoveNext(); + /*IEnumerator exceptionLines = doc.Root.GetAllExceptions()?.Select(e=>e.Location.Line).Order().GetEnumerator(); + bool hasExceptions = exceptionLines.MoveNext();*/ while (curLine < doc.LinesCount && printedLines < Math.Min(visibleLines, doc.LinesCount)) { - while (hasExceptions && exceptionLines.Current < curLine) { + /*while (hasExceptions && exceptionLines.Current < curLine) { hasExceptions = exceptionLines.MoveNext(); - } + }*/ int encodedChar = 0; TextLine curTxtLine = doc.GetLine (curLine); @@ -734,13 +734,13 @@ namespace CrowEditBase #endif if (selectionNotEmpty && curLine >= selStart.Line && curLine <= selEnd.Line) fillHighlight (gr, curLine, selStart, selEnd, lineRect, SelectionBackground); - if (hasExceptions && exceptionLines.Current == curLine) { + /*if (hasExceptions && exceptionLines.Current == curLine) { gr.Operator = Operator.DestOver; gr.SetSource (new Color(1.0,0,0.0,0.3)); gr.Rectangle (lineRect); gr.Fill (); gr.Operator = Operator.Over; - } + }*/ //Draw line numbering if (printLineNumbers){ @@ -751,7 +751,7 @@ namespace CrowEditBase if (tokPtr + 1 == doc.Tokens.Length && curLine < doc.LinesCount-1) drawLineNumber (gr, curLine+1, marginRect.X + leftMarginGap + lineNumWidth, marginRect.Y + lineHeight + fe.Ascent); } - bool curFoldStart = notEndOfFolds && foldsEnum.Current.StartLine == curLine; + bool curFoldStart = notEndOfFolds && foldsEnum.Current.StartLocation.Line == curLine; //draw fold if (curFoldStart) { Rectangle rFld = new Rectangle ((int)marginRect.Right - leftMarginGap - foldSize, @@ -783,13 +783,13 @@ namespace CrowEditBase pixY += lineHeight; printedLines++; - if (curFoldStart && curLine == foldsEnum.Current.StartLine){ + if (curFoldStart && curLine == foldsEnum.Current.StartLocation.Line){ if (foldsEnum.Current.isFolded) curLine += foldsEnum.Current.LineCount; else curLine++; notEndOfFolds = foldsEnum.MoveNext(); - while (notEndOfFolds && foldsEnum.Current.StartLine < curLine) { + while (notEndOfFolds && foldsEnum.Current.StartLocation.Line < curLine) { /*if (foldsEnum.Current.isFolded && curLine <= foldsEnum.Current.EndLine) curLine += foldsEnum.Current.LineCount;*/ notEndOfFolds = foldsEnum.MoveNext(); diff --git a/CrowEditBase/src/TextDocument.cs b/CrowEditBase/src/TextDocument.cs index 695f724..fec865e 100644 --- a/CrowEditBase/src/TextDocument.cs +++ b/CrowEditBase/src/TextDocument.cs @@ -12,6 +12,14 @@ using static CrowEditBase.CrowEditBase; namespace CrowEditBase { + public class ReadOnlyTextBuffer { + public readonly ReadOnlyMemory Source; + public readonly LineCollection Lines; + public ReadOnlyTextBuffer(ReadOnlyMemory source, LineCollection lines) { + Source = source; + Lines = lines; + } + } public class TextDocument : Document { public TextDocument (string fullPath, string editorPath = "default") : base (fullPath, editorPath) { @@ -20,8 +28,8 @@ namespace CrowEditBase protected TextBuffer buffer; public ReadOnlySpan source => buffer.ReadOnlySpan; - public ReadOnlyMemory ImmutableBufferCopy => buffer.ReadOnlyCopy; - internal LineCollection Lines => buffer.GetLineListCopy(); + + public ReadOnlyTextBuffer ImmutableBufferCopy => new ReadOnlyTextBuffer(buffer.ReadOnlyCopy, buffer.GetLineListCopy()); System.Text.Encoding encoding = System.Text.Encoding.UTF8; public event EventHandler TextChanged; diff --git a/plugins/CECrowPlugin/src/Parsing/IML/ImlSyntaxNodes.cs b/plugins/CECrowPlugin/src/Parsing/IML/ImlSyntaxNodes.cs index 7885d98..406fc01 100644 --- a/plugins/CECrowPlugin/src/Parsing/IML/ImlSyntaxNodes.cs +++ b/plugins/CECrowPlugin/src/Parsing/IML/ImlSyntaxNodes.cs @@ -11,12 +11,12 @@ namespace CECrowPlugin public class AttributeSyntax : SyntaxNode { - internal int? name, equal, valueOpen, valueClose, valueTok; + /*internal int? name, equal, valueOpen, valueClose, valueTok; public string Name => name.HasValue ? Root.GetTokenStringByIndex (TokenIndexBase + name.Value) : null; public string Value => valueTok.HasValue ? Root.GetTokenStringByIndex (TokenIndexBase + valueTok.Value) : null; public Token? ValueToken => valueTok.HasValue ? Root.GetTokenByIndex (TokenIndexBase + valueTok.Value) : null; public AttributeSyntax (int startLine, int tokenBase) : base (startLine, tokenBase) {} - public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue; + public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue;*/ } } \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/Parsing/Styling/StyleSyntaxAnalyser.cs b/plugins/CECrowPlugin/src/Parsing/Styling/StyleSyntaxAnalyser.cs index e7a670e..948c71b 100644 --- a/plugins/CECrowPlugin/src/Parsing/Styling/StyleSyntaxAnalyser.cs +++ b/plugins/CECrowPlugin/src/Parsing/Styling/StyleSyntaxAnalyser.cs @@ -22,13 +22,13 @@ namespace CECrowPlugin.Style public override async Task Process () { Tokenizer tokenizer = new StyleTokenizer(); - ReadOnlyMemory source = document.ImmutableBufferCopy; - Token[] tokens = tokenizer.Tokenize(source.Span); - currentNode = Root = new StyleRootSyntax (source, tokens); + ReadOnlyTextBuffer buff = document.ImmutableBufferCopy; + Token[] tokens = tokenizer.Tokenize(buff.Source.Span); + currentNode = Root = new StyleRootSyntax (buff, tokens); currentLine = 0; tokIdx = 0; - + /* while (tokIdx < tokens.Length) { if (!skipTrivia(true)) break; @@ -64,7 +64,7 @@ namespace CECrowPlugin.Style else currentNode = currentNode.Parent; } - setCurrentNodeEndLine (currentLine); + setCurrentNodeEndLine (currentLine);*/ return Root; } diff --git a/plugins/CECrowPlugin/src/Parsing/Styling/SyntaxNodes.cs b/plugins/CECrowPlugin/src/Parsing/Styling/SyntaxNodes.cs index a615950..3e1d8d8 100644 --- a/plugins/CECrowPlugin/src/Parsing/Styling/SyntaxNodes.cs +++ b/plugins/CECrowPlugin/src/Parsing/Styling/SyntaxNodes.cs @@ -11,41 +11,15 @@ namespace CECrowPlugin.Style { public class StyleRootSyntax : SyntaxRootNode { - public StyleRootSyntax (ReadOnlyMemory source, Token[] tokens) : base (source, tokens) { } + public StyleRootSyntax (ReadOnlyTextBuffer source, Token[] tokens) : base (source, tokens) { } } public class ConstantDefinitionSyntax : SyntaxNode { - internal int? equal; - public readonly StyleIdentifierSyntax Identifier; - public readonly ImlValueSyntax Value; - public override bool IsComplete => base.IsComplete & Identifier != null & equal.HasValue; - public ConstantDefinitionSyntax (StyleIdentifierSyntax id) - : base (id.StartLine, id.TokenIndexBase) { - Identifier = id; - } } public class StyleIdentifierSyntax : SyntaxNode { - public StyleIdentifierSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase, tokenBase) { - EndLine = startLine; - } } public class ImlValueSyntax : SyntaxNode { - internal int? valueOpen, value, valueClose; - public override bool IsComplete => base.IsComplete & value.HasValue & - valueOpen.HasValue & valueClose.HasValue; - public ImlValueSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } public class AttributeSyntax : SyntaxNode { - public Token? NameToken { get; internal set; } - public Token? EqualToken { get; internal set; } - public Token? ValueOpenToken { get; internal set; } - public Token? ValueCloseToken { get; internal set; } - public Token? ValueToken { get; internal set; } - public AttributeSyntax (int startLine, int startTok) : base (startLine, startTok) {} - public override bool IsComplete => base.IsComplete & NameToken.HasValue & EqualToken.HasValue & - ValueToken.HasValue & ValueOpenToken.HasValue & ValueCloseToken.HasValue; } } \ No newline at end of file diff --git a/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxAnalyser.cs b/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxAnalyser.cs index bfa4a9b..86759e6 100644 --- a/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxAnalyser.cs +++ b/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxAnalyser.cs @@ -25,9 +25,9 @@ namespace CrowEdit.Ebnf public override async Task Process() { Tokenizer tokenizer = new EbnfTokenizer(); - ReadOnlyMemory source = document.ImmutableBufferCopy; - Token[] tokens = tokenizer.Tokenize(source.Span); - currentNode = Root = new EbnfRootSyntax (source, tokens); + ReadOnlyTextBuffer buff = document.ImmutableBufferCopy; + Token[] tokens = tokenizer.Tokenize(buff.Source.Span); + currentNode = Root = new EbnfRootSyntax (buff, tokens); currentLine = 0; tokIdx = 0; @@ -78,10 +78,8 @@ namespace CrowEdit.Ebnf } tokIdx++; - }*/ - - - setCurrentNodeEndLine (currentLine); + } + setCurrentNodeEndLine (currentLine);*/ return Root; } diff --git a/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxNodes.cs b/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxNodes.cs index b709ffd..b5ac699 100644 --- a/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxNodes.cs +++ b/plugins/CEEbnfPlugin/src/Parsing/EbnfSyntaxNodes.cs @@ -11,82 +11,39 @@ namespace CrowEdit.Ebnf { public class EbnfRootSyntax : SyntaxRootNode { - public EbnfRootSyntax (ReadOnlyMemory source, Token[] tokens) : base (source, tokens) { } + public EbnfRootSyntax (ReadOnlyTextBuffer source, Token[] tokens) : base (source, tokens) { } } public class EbnfSyntaxNode : SyntaxNode { - public EbnfSyntaxNode(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } public class ProductionSyntax : SyntaxNode { - public int? ncname, equal; - public ExpressionSyntax expression; - public ProductionSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } - public ProductionSyntax(int name, int startLine, int tokenBase) - : base (startLine, tokenBase) { - ncname = name; - } - public override bool IsComplete => base.IsComplete; } public class ExpressionSyntax : SyntaxNode { - public ExpressionSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { } } public class LinkSyntax : ExpressionSyntax { - public int? openBracket, closingBracket; - public LinkSyntax(int openBracket, int startLine, int tokenBase) - : base (startLine, tokenBase) { - this.openBracket = openBracket; - } } public class ChoiceSyntax : ExpressionSyntax { - public ChoiceSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } // (Item ( '-' Item | Item* ))? public class SequenceOrDifferenceSyntax : SyntaxNode { - public SequenceOrDifferenceSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { } } // Item ::= Primary ( '?' | '*' | '+' )? */ public class ItemSyntax : SyntaxNode { - public ItemSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { } } /* NCName | StringLiteral | CharCode | CharClass | '(' Choice ')' */ public class PrimarySyntax : SyntaxNode { - public PrimarySyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } // StringLiteral ::= '"' [^"]* '"' | "'" [^']* "'" public class StringLiteralSyntax : SyntaxNode { - public StringLiteralSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - - } } // CharCode ::= '#x' [0-9a-fA-F]+ public class CharCodeSyntax : SyntaxNode { - public CharCodeSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } // CharClass ::= '[' '^'? ( Char | CharCode | CharRange | CharCodeRange )+ ']' public class CharClassSyntax : SyntaxNode { - public int? closingBracket; - public CharClassSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) {} } // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. public class CharSyntax : SyntaxNode { - public CharSyntax(int startLine, int tokenBase) - : base (startLine, tokenBase) { - } } diff --git a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs index a2e1fd5..dfb7f37 100644 --- a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs +++ b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs @@ -13,13 +13,9 @@ using Microsoft.CodeAnalysis.Text; namespace CERoslynPlugin { public class CSRootSyntax : SyntaxRootNode { - public CSRootSyntax (ReadOnlyMemory source, Token[] tokens) : base (source, tokens) { } + public CSRootSyntax (ReadOnlyTextBuffer source, Token[] tokens) : base (source, tokens) { } } - public class CSSyntaxNode : CrowEditBase.SyntaxNode { - - public CSSyntaxNode (int startLine, int tokenBase, int? lastTokenIdx = null) - : base (startLine, tokenBase, lastTokenIdx) { - } + public class CSSyntaxNode : MultiNodeSyntax { } public class CSSyntaxAnalyser : SyntaxAnalyser { @@ -34,9 +30,9 @@ namespace CERoslynPlugin public override async Task Process () { CSTokenizer tokenizer = new CSTokenizer(csdoc.tree); - ReadOnlyMemory source = document.ImmutableBufferCopy; + ReadOnlyTextBuffer buff = document.ImmutableBufferCopy; Token[] tokens = tokenizer.Tokenize(); - CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (source, tokens)); + CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (buff, tokens)); bridge.Visit(await tokenizer.syntaxTree.GetRootAsync()); @@ -47,7 +43,7 @@ namespace CERoslynPlugin class CsharpSyntaxWalkerBridge : Microsoft.CodeAnalysis.CSharp.CSharpSyntaxWalker { public CSRootSyntax Root; - CrowEditBase.SyntaxNode currentNode; + MultiNodeSyntax currentNode; public CsharpSyntaxWalkerBridge (CSRootSyntax root) : base (SyntaxWalkerDepth.StructuredTrivia) { currentNode = Root = root; @@ -60,11 +56,19 @@ namespace CERoslynPlugin int indexBase = Root.FindTokenIndexIncludingPosition(node.Span.Start); int lastTokIndex = Root.FindTokenIndexIncludingPosition(node.Span.End - 1); - currentNode = currentNode.AddChild(new CSSyntaxNode(loc.GetLineSpan().StartLinePosition.Line, indexBase, lastTokIndex)); + currentNode = currentNode.AddChild(new CSSyntaxNode()) as MultiNodeSyntax; base.Visit (node); - currentNode.EndLine = end.Line; + currentNode = currentNode.Parent; } + + public override void VisitToken(SyntaxToken token) + { + TextSpan fs = token.FullSpan; + Token tok = new Token(fs.Start,fs.Length,(TokenType)token.RawKind); + currentNode.AddChild(new SingleTokenSyntax(tok)); + base.VisitToken(token); + } } } \ No newline at end of file diff --git a/plugins/CEXmlPlugin/src/Parsing/XmlDocument.cs b/plugins/CEXmlPlugin/src/Parsing/XmlDocument.cs index bfe5d68..4fdc81f 100644 --- a/plugins/CEXmlPlugin/src/Parsing/XmlDocument.cs +++ b/plugins/CEXmlPlugin/src/Parsing/XmlDocument.cs @@ -32,7 +32,7 @@ namespace CrowEdit.Xml protected virtual IEnumerable getAttributeNameSuggestions(string eltName, string attribName, TextChange change) => null; protected virtual IEnumerable getAttributeValueSuggestions(string eltName, string attribName, string attribValue, TextChange change) => null; public override IList GetSuggestions (int absoluteTextPos, int currentTokenIndex, SyntaxNode CurrentNode, CharLocation loc) { - Token tok = GetTokenByIndex(currentTokenIndex); + /*Token tok = GetTokenByIndex(currentTokenIndex); if (tok.Start != absoluteTextPos //middle of edited tok && currentTokenIndex >= CurrentNode?.Root.TokenCount - 1) //occurs when curTok is last tok of text { @@ -119,7 +119,7 @@ namespace CrowEdit.Xml } - /*if (tokType == XmlTokenType.AttributeName) { + *if (tokType == XmlTokenType.AttributeName) { if (attrib.ValueToken.HasValue) { change = new TextChange (tok.Start, tok.Length, selectedSugg); newSelection = new TextSpan( @@ -141,8 +141,8 @@ namespace CrowEdit.Xml else if (tokType == XmlTokenType.AttributeValue) change = new TextChange (tok.Start, tok.Length, selectedSugg); newSelection = TextSpan.FromStartAndLength (change.End2 + offset); - }*/ - } + }* + }*/ return null; } diff --git a/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxAnalyser.cs b/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxAnalyser.cs index cf47c95..3c9da55 100644 --- a/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxAnalyser.cs +++ b/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxAnalyser.cs @@ -13,18 +13,18 @@ namespace CrowEdit.Xml public class XmlSyntaxAnalyser : SyntaxAnalyser { public XmlSyntaxAnalyser (XmlDocument document) : base (document) {} public virtual void ProcessAttributeValueSyntax(AttributeSyntax attrib) { - attrib.valueTok = tokIdx - attrib.TokenIndexBase; + //attrib.valueTok = tokIdx - attrib.TokenIndexBase; } public override async Task Process () { Tokenizer tokenizer = new XmlTokenizer(); - ReadOnlyMemory source = document.ImmutableBufferCopy; - Token[] tokens = tokenizer.Tokenize(source.Span); - currentNode = Root = new XMLRootSyntax (source, tokens); + ReadOnlyTextBuffer buff = document.ImmutableBufferCopy; + Token[] tokens = tokenizer.Tokenize(buff.Source.Span); + currentNode = Root = new XMLRootSyntax (buff, tokens); currentLine = 0; tokIdx = 0; - while (tokIdx < tokens.Length) { + /*while (tokIdx < tokens.Length) { if (curTok.Type == TokenType.LineBreak) currentLine++; else if (!curTok.Type.HasFlag (TokenType.Trivia)) { @@ -89,24 +89,6 @@ namespace CrowEdit.Xml addException ("Open/Close element name mismatch"); } finishCurrentNode (); - /*else { - addException ("Open/Close element name mismatch"); - finishCurrentNode ();//finish eltEndTag->curNode is parent elt - currentNode.RemoveChild(eltEndTag); - finishCurrentNode (-eltEndTag.TokenCount); //dont credit parent element with those tokens from the non matching end tag - //curNode should be parent element of previous element - while(currentNode is ElementSyntax esp) { - //eltEndTag is out of tree, so Name get threw exception - if (string.Equals(esp.StartTag.Name, eltEndTagName, StringComparison.Ordinal)) { - esp.EndTag = eltEndTag; - esp.AddChild (eltEndTag); - finishCurrentNode (); - break; - } else { - finishCurrentNode (-eltEndTag.TokenCount); - } - } - }*/ } else { addException ("Unexpected Token"); finishCurrentNode (-1); @@ -151,7 +133,7 @@ namespace CrowEdit.Xml currentNode = currentNode.Parent; } //check why this is required.. - setCurrentNodeEndLine (currentLine); + setCurrentNodeEndLine (currentLine);*/ return Root; } } diff --git a/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxNodes.cs b/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxNodes.cs index dca4c1d..7ae0b16 100644 --- a/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxNodes.cs +++ b/plugins/CEXmlPlugin/src/Parsing/XmlSyntaxNodes.cs @@ -10,65 +10,41 @@ namespace CrowEdit.Xml { public class XMLRootSyntax : SyntaxRootNode { - public XMLRootSyntax (ReadOnlyMemory source, Token[] tokens) : base (source, tokens) { } + public XMLRootSyntax (ReadOnlyTextBuffer buff, Token[] tokens) : base (buff, tokens) { } } - public class ProcessingInstructionSyntax : SyntaxNode { - public int? PIClose, name; - public override bool IsComplete => base.IsComplete & name.HasValue & PIClose.HasValue; - public ProcessingInstructionSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) { - } + public class ProcessingInstructionSyntax : MultiNodeSyntax { +// public override bool IsComplete => base.IsComplete & name.HasValue & PIClose.HasValue; + public ProcessingInstructionSyntax (){} } public abstract class ElementTagSyntax : SyntaxNode { - public int? name, close; - public override bool IsComplete => base.IsComplete & name.HasValue & close.HasValue; - public string Name => name.HasValue ? - Root.GetTokenStringByIndex (TokenIndexBase + name.Value) : null; - protected ElementTagSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) { - } +// public override bool IsComplete => base.IsComplete & name.HasValue & close.HasValue; + protected ElementTagSyntax () { } } public class ElementStartTagSyntax : ElementTagSyntax { - public ElementStartTagSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) { - } + public ElementStartTagSyntax () {} } public class ElementEndTagSyntax : ElementTagSyntax { - public ElementEndTagSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) { - } + public ElementEndTagSyntax () { } } - public class EmptyElementSyntax : SyntaxNode { - public readonly ElementStartTagSyntax StartTag; - public EmptyElementSyntax (ElementStartTagSyntax startNode) : base (startNode.StartLine, startNode.TokenIndexBase, startNode.LastTokenIndex) { - StartTag = startNode; - AddChild (StartTag); + public class EmptyElementSyntax : MultiNodeSyntax { + public EmptyElementSyntax (ElementStartTagSyntax startNode) { + AddChild (startNode); } - public override bool IsComplete => base.IsComplete && StartTag != null; + //public override bool IsComplete => base.IsComplete && StartTag != null; } - public class ElementSyntax : SyntaxNode { - public readonly ElementStartTagSyntax StartTag; - public ElementEndTagSyntax EndTag { get; set; } + public class ElementSyntax : MultiNodeSyntax { - public override bool IsComplete => base.IsComplete & StartTag.IsComplete & (EndTag != null && EndTag.IsComplete); + //public override bool IsComplete => base.IsComplete & StartTag.IsComplete & (EndTag != null && EndTag.IsComplete); - public ElementSyntax (ElementStartTagSyntax startTag) - : base (startTag.StartLine, startTag.TokenIndexBase) { - StartTag = startTag; - AddChild (StartTag); + public ElementSyntax (ElementStartTagSyntax startTag) { + AddChild (startTag); } } - public class AttributeSyntax : SyntaxNode { - public int? name, equal, valueOpen, valueClose, valueTok; - public string Name => name.HasValue ? Root.GetTokenStringByIndex (TokenIndexBase + name.Value) : null; - public string Value => valueTok.HasValue ? Root.GetTokenStringByIndex (TokenIndexBase + valueTok.Value) : null; - public Token? ValueToken => valueTok.HasValue ? Root.GetTokenByIndex (TokenIndexBase + valueTok.Value) : null; - public AttributeSyntax (int startLine, int tokenBase) - : base (startLine, tokenBase) {} - public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue; + public class AttributeSyntax : MultiNodeSyntax { + //public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue; } } \ No newline at end of file diff --git a/plugins/CEXmlPlugin/src/Parsing/XmlTokenType.cs b/plugins/CEXmlPlugin/src/Parsing/XmlTokenType.cs index afd7643..18d4c49 100644 --- a/plugins/CEXmlPlugin/src/Parsing/XmlTokenType.cs +++ b/plugins/CEXmlPlugin/src/Parsing/XmlTokenType.cs @@ -18,23 +18,26 @@ namespace CrowEdit.Xml BlockComment = 0x0105, BlockCommentEnd = 0x0106, Name = 0x0200, - ElementName = 0x0201, - AttributeName = 0x0202, - PI_Target = 0x0203, + ElementName = 0x8201, + AttributeName = 0x8202, + PI_Target = 0x8203, Punctuation = 0x0400, - PI_Start = 0x0401,// '' - ElementOpen = 0x0403,// '<' - EndElementOpen = 0x0404,// '' - ClosingSign = 0x0406,// '>' - DTDObjectOpen = 0x04A0,// '' + ElementOpen = 0x8403,// '<' + EndElementOpen = 0x8404,// '' + ClosingSign = 0x8406,// '>' + DTDObjectOpen = 0x84A0,// '