From: Jean-Philippe Bruyère Date: Thu, 7 Oct 2021 15:39:12 +0000 (+0000) Subject: store token indices in nodes instead of token's value, will be easier to update rando... X-Git-Url: https://git.osiis.dedyn.io/?a=commitdiff_plain;h=fd13e9f2275f87fd202cadfaaac94e2f0dcda0c7;p=jp%2Fcrowedit.git store token indices in nodes instead of token's value, will be easier to update random nodes instead of full tree --- diff --git a/CrowEditBase/src/Compiler/SourceDocument.cs b/CrowEditBase/src/Compiler/SourceDocument.cs index 58a6f11..753ebb9 100644 --- a/CrowEditBase/src/Compiler/SourceDocument.cs +++ b/CrowEditBase/src/Compiler/SourceDocument.cs @@ -47,7 +47,7 @@ namespace CrowEditBase return null; SyntaxNode sn = RootNode.FindNodeIncludingPosition (pos); if (outerMost) { - while (sn.Parent != RootNode && sn.StartToken.Start == sn.Parent.StartToken.Start) + while (sn.Parent != RootNode && sn.Span.Start == sn.Parent.Span.Start) sn = sn.Parent; } return sn; diff --git a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs index d48fc08..371a8f3 100644 --- a/CrowEditBase/src/Compiler/SyntaxAnalyser.cs +++ b/CrowEditBase/src/Compiler/SyntaxAnalyser.cs @@ -23,15 +23,16 @@ namespace CrowEditBase } public abstract void Process (); protected SyntaxNode currentNode; + protected int currentLine, tokIdx; /// /// set current node endToken and line count and set current to current.parent. /// /// The final token of this node /// the endline number of this node - protected void storeCurrentNode (Token endToken, int endLine) { - currentNode.EndToken = endToken; - currentNode.EndLine = endLine; + protected void storeCurrentNode (int endTokenOffsetFromCurrentTokIdx = 0) { + currentNode.LastTokenOffset = tokIdx - currentNode.TokenIndexBase + endTokenOffsetFromCurrentTokIdx; + currentNode.EndLine = currentLine; currentNode = currentNode.Parent; } protected void setCurrentNodeEndLine (int endLine) diff --git a/CrowEditBase/src/Compiler/SyntaxNode.cs b/CrowEditBase/src/Compiler/SyntaxNode.cs index c0080ee..2fbcd6d 100644 --- a/CrowEditBase/src/Compiler/SyntaxNode.cs +++ b/CrowEditBase/src/Compiler/SyntaxNode.cs @@ -9,11 +9,12 @@ using Crow.Text; namespace CrowEditBase { public abstract class SyntaxRootNode : SyntaxNode { - protected readonly SourceDocument source; - public SyntaxRootNode (SourceDocument source) - : base (0, source.Tokens.FirstOrDefault (), source.Tokens.LastOrDefault ()) { + internal readonly SourceDocument source; + public SyntaxRootNode (SourceDocument source) { this.source = source; } + public override int TokenIndexBase => 0; + public override int? LastTokenOffset { get => source.Tokens.Length - 1; set {} } public override SyntaxRootNode Root => this; public override bool IsFoldable => false; public override SyntaxNode NextSiblingOrParentsNextSibling => null; @@ -23,14 +24,14 @@ namespace CrowEditBase public SyntaxNode Parent { get; private set; } public int StartLine { get; private set; } public virtual int LineCount => lineCount; - public virtual bool IsComplete => EndToken.HasValue; + public virtual bool IsComplete => LastTokenOffset.HasValue; public virtual bool IsFoldable => Parent.StartLine != StartLine && lineCount > 1; public virtual SyntaxRootNode Root => Parent.Root; public virtual void UnfoldToTheTop () { isFolded = false; Parent.UnfoldToTheTop (); } - + protected Token getTokenByIndex (int idx) => Root.source.Tokens[idx]; List children = new List (); public IEnumerable Children => children; public bool HasChilds => children.Count > 0; @@ -79,12 +80,14 @@ namespace CrowEditBase } } - public readonly Token StartToken; - public Token? EndToken { get; set; } - public SyntaxNode (int startLine, Token tokStart, Token? tokEnd = null) { + public virtual int TokenIndexBase { get; private set; } + public virtual int? LastTokenOffset { get; set; } + internal SyntaxNode () {} + public SyntaxNode (int startLine, int tokenBase, int? lastTokenIdx = null) { StartLine = startLine; - StartToken = tokStart; - EndToken = tokEnd; + TokenIndexBase = tokenBase; + if (lastTokenIdx.HasValue) + LastTokenOffset = lastTokenIdx - tokenBase; } internal bool isFolded; internal int lineCount; @@ -99,7 +102,8 @@ namespace CrowEditBase /*if (HasChilds) { return new TextSpan (children.First().Span.Start, children.Last().Span.End) }*/ - return new TextSpan (StartToken.Start, EndToken.HasValue ? EndToken.Value.End : StartToken.End); + Token startTok = getTokenByIndex(TokenIndexBase); + return new TextSpan (startTok.Start, LastTokenOffset.HasValue ? getTokenByIndex (TokenIndexBase+LastTokenOffset.Value).End : startTok.End); } } @@ -128,14 +132,12 @@ namespace CrowEditBase return this is T tt ? tt : default; } - public bool Contains (int pos) => - EndToken.HasValue ? - StartToken.Start <= pos && EndToken.Value.End >= pos : false; + public bool Contains (int pos) => Span.Contains (pos); public void Dump (int level = 0) { Console.WriteLine ($"{new string('\t', level)}{this}"); foreach (SyntaxNode node in children) node.Dump (level + 1); } - public override string ToString() => $"{this.GetType().Name}: lines:({StartLine},{LineCount}) tokens:{StartToken} -> {EndToken}"; + public override string ToString() => $"{this.GetType().Name}: lines:({StartLine},{LineCount}) tokens:{TokenIndexBase} -> {LastTokenOffset}"; } } \ No newline at end of file diff --git a/plugins/CECrowPlugin/src/ImlDocument.cs b/plugins/CECrowPlugin/src/ImlDocument.cs index 479f1f5..1b4c1ad 100644 --- a/plugins/CECrowPlugin/src/ImlDocument.cs +++ b/plugins/CECrowPlugin/src/ImlDocument.cs @@ -50,7 +50,7 @@ namespace CECrowPlugin Console.WriteLine ($"Current Token: {currentToken} Current Node: {currentNode}"); #endif - if (currentToken.GetTokenType() == XmlTokenType.ElementOpen) + /*if (currentToken.GetTokenType() == XmlTokenType.ElementOpen) return new List (allWidgetNames); if (currentToken.GetTokenType() == XmlTokenType.ElementName) return allWidgetNames.Where (s => s.StartsWith (currentToken.AsString (Source), StringComparison.OrdinalIgnoreCase)).ToList (); @@ -110,14 +110,7 @@ namespace CECrowPlugin //else if (currentToken.Type == TokenType.ElementName) // Suggestions = getAllCrowTypeMembers (eltStartTag.NameToken.Value.AsString (Source)).ToList (); } else { - /*SyntaxNode curNode = source.FindNodeIncludingPosition (pos); - Console.WriteLine ($"Current Node: {curNode}"); - if (curNode is ElementStartTagSyntax eltStartTag && - (currentToken.Type != TokenType.ClosingSign && currentToken.Type != TokenType.EmptyElementClosing && currentToken.Type != TokenType.Unknown)) { - Suggestions = getAllCrowTypeMembers (eltStartTag.NameToken.Value.AsString (Source)).ToList (); - } else*/ - - } + }*/ return null; } public override TextChange? GetCompletionForCurrentToken (object suggestion, out TextSpan? newSelection) { diff --git a/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs b/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs index 043694b..d3cf493 100644 --- a/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs +++ b/plugins/CECrowPlugin/src/StyleParsing/SyntaxAnalyser.cs @@ -37,8 +37,8 @@ namespace CECrowPlugin notEndOfSource = iter.MoveNext (); } while (currentNode.Parent != null) { - if (!currentNode.EndToken.HasValue) - storeCurrentNode (previousTok, currentLine); + if (!currentNode.LastTokenOffset.HasValue) + storeCurrentNode (-1); else currentNode = currentNode.Parent; } diff --git a/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs b/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs index 2c7b73c..1f2f33e 100644 --- a/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs +++ b/plugins/CECrowPlugin/src/StyleParsing/SyntaxNodes.cs @@ -22,7 +22,7 @@ namespace CECrowPlugin public Token? ValueOpenToken { get; internal set; } public Token? ValueCloseToken { get; internal set; } public Token? ValueToken { get; internal set; } - public AttributeSyntax (int startLine, Token startTok) : base (startLine, startTok) {} + public AttributeSyntax (int startLine, int startTok) : base (startLine, startTok) {} public override bool IsComplete => base.IsComplete & NameToken.HasValue & EqualToken.HasValue & ValueToken.HasValue & ValueOpenToken.HasValue & ValueCloseToken.HasValue; } diff --git a/plugins/CEXmlPlugin/src/Parsing/SyntaxAnalyser.cs b/plugins/CEXmlPlugin/src/Parsing/SyntaxAnalyser.cs index 2a0e8d5..128a569 100644 --- a/plugins/CEXmlPlugin/src/Parsing/SyntaxAnalyser.cs +++ b/plugins/CEXmlPlugin/src/Parsing/SyntaxAnalyser.cs @@ -15,118 +15,113 @@ namespace CrowEdit.Xml } Token previousTok; - IEnumerator iter; - - int currentLine; + Token curTok => source.Tokens[tokIdx]; public override void Process () { XmlDocument xmlDoc = source as XmlDocument; Exceptions = new List (); currentNode = new IMLRootSyntax (xmlDoc); previousTok = default; - iter = xmlDoc.Tokens.AsEnumerable().GetEnumerator (); currentLine = 0; + Token[] toks = source.Tokens; + tokIdx = 0; - bool notEndOfSource = iter.MoveNext (); - while (notEndOfSource) { - if (iter.Current.Type == TokenType.LineBreak) + while (tokIdx < toks.Length) { + if (curTok.Type == TokenType.LineBreak) currentLine++; - else if (!iter.Current.Type.HasFlag (TokenType.Trivia)) { + else if (!curTok.Type.HasFlag (TokenType.Trivia)) { if (currentNode is ElementStartTagSyntax tag) { - if (iter.Current.GetTokenType() == XmlTokenType.AttributeName) { - AttributeSyntax attribute = new AttributeSyntax (currentLine, iter.Current); - attribute.NameToken = iter.Current; + if (curTok.GetTokenType() == XmlTokenType.AttributeName) { + AttributeSyntax attribute = new AttributeSyntax (currentLine, tokIdx); + attribute.name = tokIdx; currentNode = currentNode.AddChild (attribute); - } else if (iter.Current.GetTokenType() == XmlTokenType.ElementName) - tag.NameToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.ClosingSign) { - storeCurrentNode (iter.Current, currentLine); + } else if (curTok.GetTokenType() == XmlTokenType.ElementName) + tag.name = tokIdx; + else if (curTok.GetTokenType() == XmlTokenType.ClosingSign) { + storeCurrentNode (); currentNode.RemoveChild (tag); currentNode = currentNode.AddChild (new ElementSyntax (tag)); - } else if (iter.Current.GetTokenType() == XmlTokenType.EmptyElementClosing) { - storeCurrentNode (iter.Current, currentLine); + } else if (curTok.GetTokenType() == XmlTokenType.EmptyElementClosing) { + storeCurrentNode (); currentNode.RemoveChild (tag); currentNode = currentNode.AddChild (new EmptyElementSyntax (tag)); setCurrentNodeEndLine (currentLine); currentNode = currentNode.Parent; } else { - Exceptions.Add (new SyntaxException ("Unexpected Token", iter.Current)); - storeCurrentNode (previousTok, currentLine); + Exceptions.Add (new SyntaxException ("Unexpected Token", curTok)); + storeCurrentNode (-1); continue; } } else if (currentNode is ElementSyntax elt) { - if (iter.Current.GetTokenType() == XmlTokenType.ElementOpen) - currentNode = currentNode.AddChild (new ElementStartTagSyntax (currentLine, iter.Current)); - else if (iter.Current.GetTokenType() == XmlTokenType.EndElementOpen) { - elt.EndTag = new ElementEndTagSyntax (currentLine, iter.Current); + if (curTok.GetTokenType() == XmlTokenType.ElementOpen) + currentNode = currentNode.AddChild (new ElementStartTagSyntax (currentLine, tokIdx)); + else if (curTok.GetTokenType() == XmlTokenType.EndElementOpen) { + elt.EndTag = new ElementEndTagSyntax (currentLine, tokIdx); currentNode = elt.AddChild (elt.EndTag); } } else if (currentNode is AttributeSyntax attrib) { - if (iter.Current.GetTokenType() == XmlTokenType.EqualSign) - if (attrib.EqualToken.HasValue) - Exceptions.Add (new SyntaxException ("Extra equal sign in attribute syntax", iter.Current)); + if (curTok.GetTokenType() == XmlTokenType.EqualSign) + if (attrib.equal.HasValue) + Exceptions.Add (new SyntaxException ("Extra equal sign in attribute syntax", curTok)); else - attrib.EqualToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.AttributeValueOpen) - attrib.ValueOpenToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.AttributeValue) - attrib.ValueToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.AttributeValueClose) { - attrib.ValueCloseToken = iter.Current; - storeCurrentNode (iter.Current, currentLine); + attrib.equal = tokIdx - attrib.TokenIndexBase; + else if (curTok.GetTokenType() == XmlTokenType.AttributeValueOpen) + attrib.valueOpen = tokIdx - attrib.TokenIndexBase; + else if (curTok.GetTokenType() == XmlTokenType.AttributeValue) + attrib.valueTok = tokIdx - attrib.TokenIndexBase; + else if (curTok.GetTokenType() == XmlTokenType.AttributeValueClose) { + attrib.valueClose = tokIdx - attrib.TokenIndexBase; + storeCurrentNode (); } else { - Exceptions.Add (new SyntaxException ("Unexpected Token", iter.Current)); - storeCurrentNode (previousTok, currentLine); + Exceptions.Add (new SyntaxException ("Unexpected Token", curTok)); + storeCurrentNode (-1); continue; } } else if (currentNode is ElementEndTagSyntax eltEndTag) { - if (iter.Current.GetTokenType() == XmlTokenType.ElementName) - eltEndTag.NameToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.ClosingSign) { + if (curTok.GetTokenType() == XmlTokenType.ElementName) + eltEndTag.name = tokIdx - eltEndTag.TokenIndexBase; + else if (curTok.GetTokenType() == XmlTokenType.ClosingSign) { //go up 2 times - storeCurrentNode (iter.Current, currentLine); - storeCurrentNode (iter.Current, currentLine); + storeCurrentNode (); storeCurrentNode (); } else { - Exceptions.Add (new SyntaxException ("Unexpected Token", iter.Current)); - storeCurrentNode (previousTok, currentLine); - storeCurrentNode (previousTok, currentLine); + Exceptions.Add (new SyntaxException ("Unexpected Token", curTok)); + storeCurrentNode (-1); + storeCurrentNode (-1); continue; } } else if (currentNode is IMLRootSyntax) { - switch (iter.Current.GetTokenType()) { + switch (curTok.GetTokenType()) { case XmlTokenType.ElementOpen: - currentNode = currentNode.AddChild (new ElementStartTagSyntax (currentLine, iter.Current)); + currentNode = currentNode.AddChild (new ElementStartTagSyntax (currentLine, tokIdx)); break; case XmlTokenType.PI_Start: - currentNode = currentNode.AddChild (new ProcessingInstructionSyntax (currentLine, iter.Current)); + currentNode = currentNode.AddChild (new ProcessingInstructionSyntax (currentLine, tokIdx)); break; default: - Exceptions.Add (new SyntaxException ("Unexpected Token", iter.Current)); + Exceptions.Add (new SyntaxException ("Unexpected Token", curTok)); break; } } else if (currentNode is ProcessingInstructionSyntax pi) { - if (iter.Current.GetTokenType() == XmlTokenType.PI_Target) - pi.NameToken = iter.Current; - else if (iter.Current.GetTokenType() == XmlTokenType.PI_End) { - storeCurrentNode (iter.Current, currentLine); - } else if (iter.Current.GetTokenType() == XmlTokenType.AttributeName) { - AttributeSyntax attribute = new AttributeSyntax (currentLine, iter.Current); - attribute.NameToken = iter.Current; + if (curTok.GetTokenType() == XmlTokenType.PI_Target) + pi.name = tokIdx - pi.TokenIndexBase; + else if (curTok.GetTokenType() == XmlTokenType.PI_End) { + storeCurrentNode (); + } else if (curTok.GetTokenType() == XmlTokenType.AttributeName) { + AttributeSyntax attribute = new AttributeSyntax (currentLine, tokIdx); + attribute.name = 0; currentNode = currentNode.AddChild (attribute); } else { - Exceptions.Add (new SyntaxException ("Unexpected Token", iter.Current)); - storeCurrentNode (previousTok, currentLine); + Exceptions.Add (new SyntaxException ("Unexpected Token", curTok)); + storeCurrentNode (-1); continue; } } } - - previousTok = iter.Current; - notEndOfSource = iter.MoveNext (); + tokIdx++; } while (currentNode.Parent != null) { - if (!currentNode.EndToken.HasValue) - storeCurrentNode (previousTok, currentLine); + if (!currentNode.LastTokenOffset.HasValue) + storeCurrentNode (-1); else currentNode = currentNode.Parent; } diff --git a/plugins/CEXmlPlugin/src/Parsing/SyntaxNodes.cs b/plugins/CEXmlPlugin/src/Parsing/SyntaxNodes.cs index dce2483..b430ffd 100644 --- a/plugins/CEXmlPlugin/src/Parsing/SyntaxNodes.cs +++ b/plugins/CEXmlPlugin/src/Parsing/SyntaxNodes.cs @@ -14,38 +14,34 @@ namespace CrowEdit.Xml } } public class ProcessingInstructionSyntax : SyntaxNode { - public Token PIStartToken => StartToken; - public Token? PIEndToken => EndToken.HasValue && EndToken.Value.GetTokenType() == XmlTokenType.PI_End ? EndToken : null; - public Token? NameToken { get; internal set; } - public override bool IsComplete => base.IsComplete & NameToken.HasValue; - public ProcessingInstructionSyntax (int startLine, Token startTok) - : base (startLine, startTok) { + internal int? PIOpen, PIClose, name; + public override bool IsComplete => base.IsComplete & name.HasValue & PIOpen.HasValue & PIClose.HasValue; + public ProcessingInstructionSyntax (int startLine, int tokenBase) + : base (startLine, tokenBase) { } } public abstract class ElementTagSyntax : SyntaxNode { - public Token OpenToken => StartToken; - public Token? NameToken { get; internal set; } - public Token? CloseToken => EndToken.HasValue && EndToken.Value.GetTokenType() == XmlTokenType.ClosingSign ? EndToken : null; - public override bool IsComplete => base.IsComplete & NameToken.HasValue & CloseToken.HasValue; - protected ElementTagSyntax (int startLine, Token startTok) - : base (startLine, startTok) { + internal int? name, close; + public override bool IsComplete => base.IsComplete & name.HasValue & close.HasValue; + protected ElementTagSyntax (int startLine, int tokenBase) + : base (startLine, tokenBase) { } } public class ElementStartTagSyntax : ElementTagSyntax { - public ElementStartTagSyntax (int startLine, Token startTok) - : base (startLine, startTok) { + public ElementStartTagSyntax (int startLine, int tokenBase) + : base (startLine, tokenBase) { } } public class ElementEndTagSyntax : ElementTagSyntax { - public ElementEndTagSyntax (int startLine, Token startTok) - : base (startLine, startTok) { + public ElementEndTagSyntax (int startLine, int tokenBase) + : base (startLine, tokenBase) { } } public class EmptyElementSyntax : SyntaxNode { public readonly ElementStartTagSyntax StartTag; - public EmptyElementSyntax (ElementStartTagSyntax startNode) : base (startNode.StartLine, startNode.StartToken, startNode.EndToken) { + public EmptyElementSyntax (ElementStartTagSyntax startNode) : base (startNode.StartLine, startNode.TokenIndexBase, startNode.LastTokenOffset) { StartTag = startNode; AddChild (StartTag); } @@ -58,19 +54,21 @@ namespace CrowEdit.Xml public override bool IsComplete => base.IsComplete & StartTag.IsComplete & (EndTag != null && EndTag.IsComplete); public ElementSyntax (ElementStartTagSyntax startTag) - : base (startTag.StartLine, startTag.StartToken) { + : base (startTag.StartLine, startTag.TokenIndexBase) { StartTag = startTag; AddChild (StartTag); } } public class AttributeSyntax : SyntaxNode { - public Token? NameToken { get; internal set; } - public Token? EqualToken { get; internal set; } - public Token? ValueOpenToken { get; internal set; } - public Token? ValueCloseToken { get; internal set; } - public Token? ValueToken { get; internal set; } - public AttributeSyntax (int startLine, Token startTok) : base (startLine, startTok) {} - public override bool IsComplete => base.IsComplete & NameToken.HasValue & EqualToken.HasValue & ValueToken.HasValue & ValueOpenToken.HasValue & ValueCloseToken.HasValue; + internal int? name, equal, valueOpen, valueClose, valueTok; + /*public Token? NameToken => name.HasValue ? getTokenByIndex (TokenIndexBase + name.Value) : default; + public int? EqualToken { get; internal set; } + public int? ValueOpenToken { get; internal set; } + public int? ValueCloseToken { get; internal set; } + public int? ValueToken { get; internal set; }*/ + public AttributeSyntax (int startLine, int tokenBase) + : base (startLine, tokenBase) {} + public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue; } } \ No newline at end of file