From: Jean-Philippe Bruyère Date: Sun, 9 Mar 2025 18:15:17 +0000 (+0100) Subject: cs parsing X-Git-Url: https://git.osiis.dedyn.io/?a=commitdiff_plain;h=89032b7dbe377523e0c5e0fcf71f5b63eafdd11a;p=jp%2Fcrowedit.git cs parsing --- diff --git a/plugins/CERoslynPlugin/src/CSDocument.cs b/plugins/CERoslynPlugin/src/CSDocument.cs index 2adc607..a270b81 100644 --- a/plugins/CERoslynPlugin/src/CSDocument.cs +++ b/plugins/CERoslynPlugin/src/CSDocument.cs @@ -71,12 +71,12 @@ namespace CERoslynPlugin public override string GetTokenTypeString (TokenType tokenType) => ((SyntaxKind)tokenType).ToString(); public override Color GetColorForToken(Token token) { - SyntaxKind tokType = (SyntaxKind)token.Type; + SyntaxKind syntaxKind = (SyntaxKind)token.Type; + TokenType tokType = token.Type; CSTokenType xmlTokType = (CSTokenType)tokType; - if (tokType == SyntaxKind.IdentifierToken) + if (syntaxKind == SyntaxKind.IdentifierToken) return Colors.Blue; - /*CSTokenType xmlTokType = (CSTokenType)tokType; if (xmlTokType.HasFlag (CSTokenType.Punctuation)) return Colors.DarkGrey; if (tokType.HasFlag (TokenType.WhiteSpace)) @@ -94,7 +94,7 @@ namespace CERoslynPlugin if (xmlTokType == CSTokenType.Directive) return Colors.Black; if (xmlTokType == CSTokenType.Operator) - return Colors.DarkSlateBlue;*/ + return Colors.DarkSlateBlue; return Colors.Red; } diff --git a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs index 1814cde..0ff1fc3 100644 --- a/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs +++ b/plugins/CERoslynPlugin/src/CSSyntaxAnalyser.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using Crow.Text; using CrowEditBase; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; @@ -15,15 +16,25 @@ using Microsoft.CodeAnalysis.Text; namespace CERoslynPlugin { public class CSRootSyntax : SyntaxRootNode { - public CSRootSyntax (ReadOnlyTextBuffer source, Token[] tokens) : base (source, tokens) { } + public CSRootSyntax (ReadOnlyTextBuffer source) : base (source, null) { } + internal void SetTokens(Token[] tokens) { + this.tokens = tokens; + } } public class CSToken : SingleTokenSyntax { SyntaxToken cstoken; public CSToken(SyntaxToken token, Token tok) : base (tok) { cstoken = token; } - public override string ToString() => $"TOK:{cstoken.Kind()}"; + public override string ToString() => $"TOK: {cstoken.Kind()}"; } + public class CSTrivia : SingleTokenSyntax { + SyntaxTrivia cstrivia; + public CSTrivia(SyntaxTrivia token, Token tok) : base (tok) { + cstrivia = token; + } + public override string ToString() => $"Trivia: {cstrivia.Kind()}"; + } public class CSSyntaxNode : MultiNodeSyntax { Microsoft.CodeAnalysis.SyntaxNode node; public CSSyntaxNode(Microsoft.CodeAnalysis.SyntaxNode node) { @@ -44,13 +55,13 @@ namespace CERoslynPlugin } public override async Task Process () { - CSTokenizer tokenizer = new CSTokenizer(csdoc.tree); ReadOnlyTextBuffer buff = document.ImmutableBufferCopy; - Token[] tokens = tokenizer.Tokenize(); - CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (buff, tokens)); - bridge.Visit(await tokenizer.syntaxTree.GetRootAsync()); + CsharpSyntaxWalkerBridge bridge = new CsharpSyntaxWalkerBridge(new CSRootSyntax (buff)); + + bridge.Visit(await csdoc.tree.GetRootAsync()); + bridge.Root.SetTokens (bridge.Toks.ToArray()); Root = bridge.Root; return Root; } @@ -58,19 +69,15 @@ namespace CERoslynPlugin class CsharpSyntaxWalkerBridge : Microsoft.CodeAnalysis.CSharp.CSharpSyntaxWalker { public CSRootSyntax Root; + public List Toks; MultiNodeSyntax currentNode; public CsharpSyntaxWalkerBridge (CSRootSyntax root) : base (SyntaxWalkerDepth.StructuredTrivia) { currentNode = Root = root; + Toks = new List(100); } public override void Visit (Microsoft.CodeAnalysis.SyntaxNode node) { - /*Location loc = node.GetLocation(); - LinePosition start = loc.GetLineSpan().StartLinePosition; - LinePosition end = loc.GetLineSpan().EndLinePosition; - - int indexBase = Root.FindTokenIndexIncludingPosition(node.Span.Start); - int lastTokIndex = Root.FindTokenIndexIncludingPosition(node.Span.End - 1);*/ currentNode = currentNode.AddChild(new CSSyntaxNode(node)) as MultiNodeSyntax; base.Visit (node); @@ -78,16 +85,92 @@ namespace CERoslynPlugin currentNode = currentNode.Parent; } - public override void VisitToken(SyntaxToken token) + /*public override void VisitToken(SyntaxToken token) { - TextSpan fs = token.Span; - if (token.Span.Length == 0) + + base.VisitToken(token); + }*/ + + public override void VisitToken (SyntaxToken token) + { + VisitLeadingTrivia (token); + + Microsoft.CodeAnalysis.Text.TextSpan fs = token.Span; + /*if (SyntaxFacts.IsLiteralExpression (token.Kind ())) + addMultilineToken(token.ToString(), token.Span, (TokenType)token.RawKind); + else*/ + if (fs.Length == 0) Debug.WriteLine($"Empty token: {token}"); else { + Microsoft.CodeAnalysis.Text.TextSpan span = token.Span; Token tok = new Token(fs.Start,fs.Length,(TokenType)token.RawKind); + Toks.Add(tok); currentNode.AddChild(new CSToken(token, tok)); } - base.VisitToken(token); - } + + VisitTrailingTrivia (token); + } + + public override void VisitTrivia (SyntaxTrivia trivia) + { + SyntaxKind kind = trivia.Kind (); + Microsoft.CodeAnalysis.Text.TextSpan span = trivia.Span; + if (kind == SyntaxKind.EndOfLineTrivia) { + Toks.Add (new Token(span.Start, span.Length, TokenType.LineBreak)); + return; + } + if (trivia.HasStructure) + this.Visit ((CSharpSyntaxNode)trivia.GetStructure()); + else if (trivia.IsKind (SyntaxKind.DisabledTextTrivia) || trivia.IsKind (SyntaxKind.MultiLineCommentTrivia)) + addMultilineToken(trivia.ToString(), trivia.Span, (TokenType)trivia.RawKind); + else { + Toks.Add (new Token(span.Start, span.Length, (TokenType)trivia.RawKind)); + } + } + int startOfTok; + void addTok (ref SpanCharReader reader, int offset, Enum tokType) { + if (reader.CurrentPosition == startOfTok) + return; + Token tok = new Token((TokenType)tokType,startOfTok + offset, reader.CurrentPosition + offset); + Toks.Add (tok); + currentNode.AddChild(new CSToken(default, tok)); + + startOfTok = reader.CurrentPosition; + } + void addMultilineToken(ReadOnlySpan txt, Microsoft.CodeAnalysis.Text.TextSpan span, TokenType mainType) { + SpanCharReader reader = new SpanCharReader(txt); + startOfTok = 0; + + while(!reader.EndOfSpan) { + switch (reader.Peek) { + case '\x85': + case '\x2028': + case '\xA': + addTok (ref reader, span.Start, mainType); + reader.Read(); + addTok (ref reader, span.Start, TokenType.LineBreak); + break; + case '\xD': + addTok (ref reader, span.Start, mainType); + reader.Read(); + if (reader.IsNextCharIn ('\xA', '\x85')) + reader.Read(); + addTok (ref reader, span.Start, TokenType.LineBreak); + break; + case '\x20': + case '\x9': + addTok (ref reader, span.Start, mainType); + char c = reader.Read(); + while (reader.TryPeek (c)) + reader.Read(); + addTok (ref reader, span.Start, c == '\x20' ? TokenType.WhiteSpace : TokenType.Tabulation); + break; + default: + reader.Read(); + break; + } + } + addTok (ref reader, span.Start, mainType); + } } } \ No newline at end of file