cs parsing

diff --git a/plugins/CERoslynPlugin/src/CSDocument.cs b/plugins/CERoslynPlugin/src/CSDocument.cs
index 2adc607e1614acb090f16617754ff79636e0b20c..a270b81aba1de9d62d1591b1a91e8a3f537450a9 100644 (file)
--- a/plugins/CERoslynPlugin/src/CSDocument.cs
+++ b/plugins/CERoslynPlugin/src/CSDocument.cs
@@ -71,12 +71,12 @@ namespace CERoslynPlugin
                public override string GetTokenTypeString (TokenType tokenType) => ((SyntaxKind)tokenType).ToString();
                public override Color GetColorForToken(Token token)
                {
-                       SyntaxKind tokType = (SyntaxKind)token.Type;
+                       SyntaxKind syntaxKind = (SyntaxKind)token.Type;
+                       TokenType tokType = token.Type;
                        CSTokenType xmlTokType = (CSTokenType)tokType;
-                       if (tokType == SyntaxKind.IdentifierToken)
+                       if (syntaxKind == SyntaxKind.IdentifierToken)
                                return Colors.Blue;
                        
-                       /*CSTokenType xmlTokType = (CSTokenType)tokType;
                        if (xmlTokType.HasFlag (CSTokenType.Punctuation))
                                return Colors.DarkGrey;
                        if (tokType.HasFlag (TokenType.WhiteSpace))
@@ -94,7 +94,7 @@ namespace CERoslynPlugin
                        if (xmlTokType == CSTokenType.Directive)
                                return Colors.Black;
                        if (xmlTokType == CSTokenType.Operator)
-                               return Colors.DarkSlateBlue;*/
+                               return Colors.DarkSlateBlue;
                        return Colors.Red;
 
                }

diff --git a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs
index 1814cde2c066df5ef4cf9c7c33303f4bc1ef8183..0ff1fc3fc9de3b2331f2b963ce1c5d57f778c4fc 100644 (file)
--- a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs
+++ b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs
@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
+using Crow.Text;
 using CrowEditBase;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
@@ -15,15 +16,25 @@ using Microsoft.CodeAnalysis.Text;
 namespace CERoslynPlugin
 {
        public class CSRootSyntax : SyntaxRootNode {
-               public CSRootSyntax (ReadOnlyTextBuffer source, Token[] tokens) : base (source, tokens) {       }
+               public CSRootSyntax (ReadOnlyTextBuffer source) : base (source, null) { }
+               internal void SetTokens(Token[] tokens) {
+                       this.tokens = tokens;
+               }
        }
        public class CSToken : SingleTokenSyntax {
                SyntaxToken cstoken;
                public CSToken(SyntaxToken token, Token tok) : base (tok) {
                        cstoken = token;
                }
-               public override string ToString() => $"TOK:{cstoken.Kind()}";
+               public override string ToString() => $"TOK: {cstoken.Kind()}";
        }
+       public class CSTrivia : SingleTokenSyntax {
+               SyntaxTrivia cstrivia;
+               public CSTrivia(SyntaxTrivia token, Token tok) : base (tok) {
+                       cstrivia = token;
+               }
+               public override string ToString() => $"Trivia: {cstrivia.Kind()}";
+       }       
        public class CSSyntaxNode : MultiNodeSyntax {
                Microsoft.CodeAnalysis.SyntaxNode node;
                public CSSyntaxNode(Microsoft.CodeAnalysis.SyntaxNode node) {
@@ -44,13 +55,13 @@ namespace CERoslynPlugin
                }
 
                public override async Task<SyntaxRootNode> Process () {
-                       CSTokenizer tokenizer = new CSTokenizer(csdoc.tree);
                        ReadOnlyTextBuffer buff = document.ImmutableBufferCopy;
-                       Token[] tokens = tokenizer.Tokenize();
-                       CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (buff, tokens));
                        
-                       bridge.Visit(await tokenizer.syntaxTree.GetRootAsync());
 
+                       CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (buff));
+                       
+                       bridge.Visit(await csdoc.tree.GetRootAsync());
+                       bridge.Root.SetTokens (bridge.Toks.ToArray());
                        Root = bridge.Root;
                        return Root;
                }
@@ -58,19 +69,15 @@ namespace CERoslynPlugin
        class CsharpSyntaxWalkerBridge : Microsoft.CodeAnalysis.CSharp.CSharpSyntaxWalker
        {
                public CSRootSyntax Root;
+               public List<Token> Toks;
                MultiNodeSyntax currentNode;
                public CsharpSyntaxWalkerBridge (CSRootSyntax root) : base (SyntaxWalkerDepth.StructuredTrivia)
                {
                        currentNode = Root = root;
+                       Toks = new List<Token>(100);
                }
                public override void Visit (Microsoft.CodeAnalysis.SyntaxNode node)
                {
-                       /*Location loc = node.GetLocation();
-                       LinePosition start = loc.GetLineSpan().StartLinePosition;
-                       LinePosition end = loc.GetLineSpan().EndLinePosition;
-
-                       int indexBase = Root.FindTokenIndexIncludingPosition(node.Span.Start);
-                       int lastTokIndex = Root.FindTokenIndexIncludingPosition(node.Span.End - 1);*/
                        currentNode = currentNode.AddChild(new CSSyntaxNode(node)) as MultiNodeSyntax;
                        
                        base.Visit (node);
@@ -78,16 +85,92 @@ namespace CERoslynPlugin
                        currentNode = currentNode.Parent;
                }
 
-        public override void VisitToken(SyntaxToken token)
+        /*public override void VisitToken(SyntaxToken token)
         {
-                       TextSpan fs = token.Span;
-                       if (token.Span.Length == 0)
+                       
+            base.VisitToken(token);
+        }*/
+
+               public override void VisitToken (SyntaxToken token)
+               {
+                       VisitLeadingTrivia (token);
+
+                       Microsoft.CodeAnalysis.Text.TextSpan fs = token.Span;
+                       /*if (SyntaxFacts.IsLiteralExpression (token.Kind ()))
+                               addMultilineToken(token.ToString(), token.Span, (TokenType)token.RawKind);
+                       else*/
+                       if (fs.Length == 0)
                                Debug.WriteLine($"Empty token: {token}");
                        else {
+                               Microsoft.CodeAnalysis.Text.TextSpan span = token.Span;
                                Token tok = new Token(fs.Start,fs.Length,(TokenType)token.RawKind);
+                               Toks.Add(tok);
                                currentNode.AddChild(new CSToken(token, tok));
                        }
-            base.VisitToken(token);
-        }
+
+                       VisitTrailingTrivia (token);
+               }
+               
+        public override void VisitTrivia (SyntaxTrivia trivia)
+               {
+                       SyntaxKind kind = trivia.Kind ();
+                       Microsoft.CodeAnalysis.Text.TextSpan span = trivia.Span;
+                       if (kind == SyntaxKind.EndOfLineTrivia) {
+                               Toks.Add (new Token(span.Start, span.Length, TokenType.LineBreak));
+                               return;
+                       }
+                       if (trivia.HasStructure)
+                               this.Visit ((CSharpSyntaxNode)trivia.GetStructure());
+                       else if (trivia.IsKind (SyntaxKind.DisabledTextTrivia) || trivia.IsKind (SyntaxKind.MultiLineCommentTrivia))
+                addMultilineToken(trivia.ToString(), trivia.Span, (TokenType)trivia.RawKind);
+                       else {
+                               Toks.Add (new Token(span.Start, span.Length, (TokenType)trivia.RawKind));
+                       }
+               }
+               int startOfTok;
+               void addTok (ref SpanCharReader reader, int offset, Enum tokType) {
+                       if (reader.CurrentPosition == startOfTok)
+                               return;
+                       Token tok = new Token((TokenType)tokType,startOfTok + offset, reader.CurrentPosition + offset);
+                       Toks.Add (tok);
+                       currentNode.AddChild(new CSToken(default, tok));
+
+                       startOfTok = reader.CurrentPosition;
+               }               
+               void addMultilineToken(ReadOnlySpan<char> txt, Microsoft.CodeAnalysis.Text.TextSpan span, TokenType mainType) {
+                       SpanCharReader reader = new SpanCharReader(txt);
+                       startOfTok = 0;
+
+                       while(!reader.EndOfSpan) {
+                               switch (reader.Peek) {
+                                       case '\x85':
+                                       case '\x2028':
+                                       case '\xA':
+                                               addTok (ref reader, span.Start, mainType);
+                                               reader.Read();
+                                               addTok (ref reader, span.Start, TokenType.LineBreak);
+                                               break;
+                                       case '\xD':
+                                               addTok (ref reader, span.Start, mainType);
+                                               reader.Read();
+                                               if (reader.IsNextCharIn ('\xA', '\x85'))
+                                                       reader.Read();
+                                               addTok (ref reader, span.Start, TokenType.LineBreak);
+                                               break;
+                                       case '\x20':
+                                       case '\x9':
+                                               addTok (ref reader, span.Start, mainType);
+                                               char c = reader.Read();
+                                               while (reader.TryPeek (c))
+                                                       reader.Read();
+                                               addTok (ref reader, span.Start, c == '\x20' ? TokenType.WhiteSpace : TokenType.Tabulation);
+                                               break;
+                                       default:
+                                               reader.Read();
+                                               break;
+                               }
+                       }                       
+                       addTok (ref reader, span.Start, mainType);
+               }                               
     }
 }
\ No newline at end of file