public abstract class SyntaxAnalyser {
protected SourceDocument document;
protected SyntaxRootNode Root;
+ protected CancellationToken cancel;
public IEnumerable<SyntaxException> Exceptions => null;// Root?.GetAllExceptions();
public SyntaxAnalyser (SourceDocument document) {
this.document = document;
#endregion
#region parsing context
- protected int currentLine = 0, tokIdx = 0;
+ protected int tokIdx = 0;
//protected MultiNodeSyntax currentNode;
#endregion
if (tok.Type == TokenType.LineBreak) {
if (!skipLineBreaks)
return true;
- currentLine++;
}
tokIdx++;
}
if (tok.Type == TokenType.LineBreak) {
if (!skipLineBreaks)
return true;
- currentLine++;
}
tokIdx++;
}
public override bool HasChilds => children.Count > 0;
public override int SpanStart => HasChilds ? children[0].SpanStart : 0;
public override int SpanEnd => HasChilds ? children[children.Count - 1].SpanEnd : 0;
+ /*public override int SpanEnd {//to be removed, used for debugging
+ get {
+ if (HasChilds) {
+
+ if (children[children.Count - 1] == this)
+ System.Diagnostics.Debugger.Break();
+ return children[children.Count - 1].SpanEnd;
+ } else return 0;
+ }
+ } */
internal bool isFolded;
public virtual bool IsFoldable => IsComplete && !(Parent != Root && Parent.StartLocation.Line == StartLocation.Line) && LineCount > 1;
#region Keyboard handling
public override void onKeyPress (object sender, KeyPressEventArgs e) {
base.onKeyPress (sender, e);
-
if (!e.Handled) {
TextSpan selection = Selection;
update (new TextChange (selection.Start, selection.Length, e.KeyChar.ToString ()));
}
}
- public Token CurrentToken => sourceDocument != null && sourceDocument.IsParsed ?
- sourceDocument.GetTokenByIndex(currentTokenIndex) : default;
+ bool parsingOk => sourceDocument != null && sourceDocument.IsParsed;
+ public Token CurrentToken => parsingOk ? sourceDocument.GetTokenByIndex(currentTokenIndex) : default;
#if DEBUG
- public string CurrentTokenString => sourceDocument != null && sourceDocument.IsParsed ?
- CurrentToken.AsString(Document.source) : null;
- public string CurrentTokenType => sourceDocument != null && CurrentToken != null && sourceDocument.IsParsed ?
- sourceDocument.GetTokenTypeString(CurrentToken.Type) : default;
+ public string CurrentTokenString {
+ get {
+ try
+ {
+ return parsingOk ? CurrentToken.AsString(Document.source) : null;
+ }
+ catch (System.Exception e)
+ {
+ Console.WriteLine($"{CurrentToken}:{e}");
+ }
+ return null;
+ }
+
+ }
+ public string CurrentTokenType {
+ get {
+ try
+ {
+ return parsingOk && CurrentToken != null ?
+ sourceDocument.GetTokenTypeString(CurrentToken.Type) : default;
+ }
+ catch (System.Exception e)
+ {
+ Console.WriteLine($"{CurrentToken}:{e}");
+ }
+ return default;
+ }
+
+ }
+ public int AbsoluteCharLoc {
+ get {
+ try
+ {
+ return currentLoc.HasValue ?
+ Document.GetAbsolutePosition(currentLoc.Value) : 0;
+ }
+ catch (System.Exception e)
+ {
+ Console.WriteLine (e);
+ }
+ return 0;
+ }
+ }
#endif
#region suggestions and autocomplete
if (currentLoc == value)
return;
currentLoc = value;
- if (currentLoc.HasValue) {
+ if (currentLoc.HasValue && Document is SourceDocument doc && doc.IsParsed) {
MultiNodeSyntax fold = getFoldContainingLine (currentLoc.Value.Line);
while (fold != null && fold.StartLocation.Line == currentLoc.Value.Line)
fold = fold.Parent;
#region Mouse & Keyboard overrides
public override void onMouseDown (object sender, MouseButtonEventArgs e) {
hideOverlay ();
- if (mouseIsInMargin) {
- if (e.Button == MouseButton.Left && mouseIsInFoldRect) {
- MultiNodeSyntax curNode = getFoldStartingAt (hoverLoc.Value.Line);
- if (curNode != null) {
- curNode.isFolded = !curNode.isFolded;
- textMeasureIsUpToDate = false;
- RegisterForRedraw();
+ if (parsingOk) {
+ if (mouseIsInMargin) {
+ if (e.Button == MouseButton.Left && mouseIsInFoldRect) {
+ MultiNodeSyntax curNode = getFoldStartingAt (hoverLoc.Value.Line);
+ if (curNode != null) {
+ curNode.isFolded = !curNode.isFolded;
+ textMeasureIsUpToDate = false;
+ RegisterForRedraw();
+ }
}
- }
- e.Handled = true;
+ e.Handled = true;
+ }
}
-
base.onMouseDown (sender, e);
}
protected override void mouseMove (MouseEventArgs e) {
+ if (!parsingOk) {
+ base.mouseMove(e);
+ return;
+ }
Point mLoc = ScreenPointToLocal (e.Position);
if (mLoc.X < leftMargin - leftMarginRightGap) {
mouseIsInMargin = true;
mouseIsInMargin = mouseIsInFoldRect = false;
IFace.MouseCursor = MouseCursor.ibeam;
}
-
- int hoverVisualLine = getLineIndexFromMousePositionUnchecked (mLoc);
+
+ //int hoverVisualLine = getLineIndexFromMousePositionUnchecked (mLoc);
+ int hoverVisualLine = getLineIndexFromMousePosition (mLoc);
int hoverLine = getAbsoluteLineFromVisualLine (hoverVisualLine);
/***************************/
int newVl = Math.Min (Math.Max (0, getVisualLineFromAboluteLine (startLine) + visualLineDiff), visualLineCount - 1);
return getAbsoluteLineFromVisualLine (newVl);
}
-
- protected override int visualLineCount
- {
- get {
- if (!(Document is SourceDocument doc))
- return base.visualLineCount;
-
- return Document.LinesCount - doc.Root.FoldedLineCount;
- }
- }
+ protected override int visualLineCount => parsingOk ? Document.LinesCount - sourceDocument.Root.FoldedLineCount : base.visualLineCount;
protected override int visualCurrentLine => CurrentLoc.HasValue ? getVisualLineFromAboluteLine (CurrentLoc.Value.Line) : 0;
protected override void updateMaxScrolls (LayoutingType layout) {
updateMargin();
}
protected override void drawContent (IContext gr) {
- if (!(Document is SourceDocument doc)) {
+ if (!parsingOk) {
base.drawContent (gr);
return;
}
- doc.EnterReadLock ();
+ sourceDocument.EnterReadLock ();
try {
double lineHeight = fe.Ascent + fe.Descent;
updateMargin ();
gr.SetSource (marginBG);
gr.Rectangle (marginRect);
gr.Fill();
-
- if (!doc.IsParsed) {
- base.drawContent (gr);
- return;
- }
gr.Translate (-ScrollX, 0);
bool showWhiteSpaces = App.ShowWhiteSpace;
string tabString = showWhiteSpaces ?
$"{new string(' ', (App.TabulationSize - 2) / 2)} \u2192{new string(' ', (App.TabulationSize - 2) / 2)}" : default;
- ReadOnlySpan<char> sourceBytes = doc.source;
+
+ ReadOnlySpan<char> sourceBytes = sourceDocument.source;
Span<byte> bytes = stackalloc byte[128];
TextExtents extents;
ReadOnlySpan<char> buff = sourceBytes;
int linesToSkip = (int)Math.Floor(ScrollY / lineHeight);
- IEnumerator<MultiNodeSyntax> foldsEnum = doc.Root.VisibleFoldableNodes.GetEnumerator ();
+ IEnumerator<MultiNodeSyntax> foldsEnum = sourceDocument.Root.VisibleFoldableNodes.GetEnumerator ();
bool notEndOfFolds = foldsEnum.MoveNext();
while (notEndOfFolds && foldsEnum.Current.StartLocation.Line < linesToSkip) {
/*IEnumerator<int> exceptionLines = doc.Root.GetAllExceptions()?.Select(e=>e.Location.Line).Order().GetEnumerator();
bool hasExceptions = exceptionLines.MoveNext();*/
- while (curLine < doc.LinesCount && printedLines < Math.Min(visibleLines, doc.LinesCount)) {
+ while (curLine < sourceDocument.LinesCount && printedLines < Math.Min(visibleLines, sourceDocument.LinesCount)) {
/*while (hasExceptions && exceptionLines.Current < curLine) {
hasExceptions = exceptionLines.MoveNext();
}*/
int encodedChar = 0;
- TextLine curTxtLine = doc.GetLine (curLine);
- int tokPtr = doc.FindTokenIndexIncludingPosition(curTxtLine.Start);
- Token tok = doc.Tokens[tokPtr];
+ TextLine curTxtLine = sourceDocument.GetLine (curLine);
+ int tokPtr = sourceDocument.FindTokenIndexIncludingPosition(curTxtLine.Start);
+ Token tok = sourceDocument.Tokens[tokPtr];
while (tok.Start < (showWhiteSpaces ? curTxtLine.EndIncludingLineBreak : curTxtLine.End)) {
if (showWhiteSpaces && tok.Type.HasFlag(TokenType.WhiteSpace)) {
gr.ShowText (buff);*/
} else
buff = sourceBytes.Slice (tok.Start, tok.Length);
- gr.SetSource (doc.GetColorForToken (tok));
+ gr.SetSource (sourceDocument.GetColorForToken (tok));
int size = buff.Length * 4 + 1;
if (bytes.Length < size)
Rectangle r = new RectangleD(pixX, pixY, extents.Width, lineHeight);
r.Inflate(1);
gr.Rectangle(r);
- gr.SetSource(doc.GetColorForToken (tok).AdjustAlpha(0.5));
+ gr.SetSource(sourceDocument.GetColorForToken (tok).AdjustAlpha(0.5));
gr.Stroke();
}
pixX += extents.XAdvance;
}
- if (++tokPtr >= doc.Tokens.Length)
+ if (++tokPtr >= sourceDocument.Tokens.Length)
break;
- tok = doc.Tokens[tokPtr];
+ tok = sourceDocument.Tokens[tokPtr];
}
RectangleD lineRect = new RectangleD (cb.X, pixY, pixX - cb.X, lineHeight);
gr.SetSource (marginFG);
drawLineNumber (gr, curLine, marginRect.X + leftMarginGap + lineNumWidth, marginRect.Y + fe.Ascent);
- if (tokPtr + 1 == doc.Tokens.Length && curLine < doc.LinesCount-1)
+ if (tokPtr + 1 == sourceDocument.Tokens.Length && curLine < sourceDocument.LinesCount-1)
drawLineNumber (gr, curLine+1, marginRect.X + leftMarginGap + lineNumWidth, marginRect.Y + lineHeight + fe.Ascent);
}
bool curFoldStart = notEndOfFolds && foldsEnum.Current.StartLocation.Line == curLine;
gr.Stroke ();
}
- if (++tokPtr >= doc.Tokens.Length)
+ if (++tokPtr >= sourceDocument.Tokens.Length)
break;
- tok = doc.Tokens[tokPtr];
+ tok = sourceDocument.Tokens[tokPtr];
pixX = cb.Left;
pixY += lineHeight;
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
} finally {
- doc.ExitReadLock ();
+ sourceDocument.ExitReadLock ();
}
gr.Translate (ScrollX, 0);
}
#if DEBUG
NotifyValueChanged("CurrentTokenString",CurrentTokenString);
NotifyValueChanged("CurrentTokenType",CurrentTokenType);
+ NotifyValueChanged("AbsoluteCharLoc",AbsoluteCharLoc);
+
#endif
<Label Text="{../../tb.CurrentLine}" Margin="3"/>
<Label Text="Col:" Foreground="Grey"/>
<Label Text="{../../tb.TabulatedColumn}" Margin="3"/>
+ <Label Text="Abs:" Foreground="Grey"/>
+ <Label Text="{../../tb.AbsoluteCharLoc}" Margin="3"/>
</HorizontalStack>
</VerticalStack>
</ListItem>
public static void SetTokenType (this Token tok, StyleTokenType type) {
tok.Type = (TokenType)type;
}
- public static bool Is(this Token tok, StyleTokenType type) => (StyleTokenType)tok.Type == type;
+ public static bool Is(this Token tok, StyleTokenType type) => (StyleTokenType)tok.Type == type;
}
public class StyleSyntaxAnalyser : SyntaxAnalyser {
public StyleSyntaxAnalyser (StyleDocument document) : base (document) {}
while (tryPeekFlag(out Token token, TokenType.Trivia)) {
switch(token.GetTokenType()) {
case (StyleTokenType)TokenType.LineBreak:
- currentLine++;
Read();
break;
case StyleTokenType.LineCommentStart:
}
if (tok.Type == TokenType.LineBreak) {
Read();
- currentLine++;
} else {
bc.AddChild(new SingleTokenSyntax(Read()));
}
accept(cst, StyleTokenType.ClosingBrace);
return cst;
}
- CancellationToken cancel;
+
public override async Task<SyntaxRootNode> Process (CancellationToken cancel = default) {
Tokenizer tokenizer = new StyleTokenizer();
ReadOnlyTextBuffer buff = document.ImmutableBufferCopy;
if (!skipTriviaAndComments(Root))
break;
if (!Peek().Is(StyleTokenType.Name)) {
- Root.AddChild(new UnexpectedTokenSyntax(Read()));
+ Root.AddChild(new UnexpectedTokenSyntax(Read()));
continue;
}
Token name = Read();
} else
reader.Read ();
}
+ } else {
+ addTok (ref reader, StyleTokenType.Unknown);
}
break;
case ',':
reader.Advance ();
addTok (ref reader, StyleTokenType.MemberValueOpen);
- while (!reader.EndOfSpan) {
+ while (!reader.Eol()) {
if (reader.TryPeek ("${")) {
addTok (ref reader, StyleTokenType.MemberValuePart);
reader.Advance (2);
addTok (ref reader, StyleTokenType.ConstantRefOpen);
- while (!reader.EndOfSpan) {
+ while (!reader.Eol()) {
if (reader.TryPeek ('}')) {
addTok (ref reader, StyleTokenType.ConstantName);
reader.Read ();
namespace CrowEdit.Ebnf
{
- public static class Extensions {
- public static EbnfTokenType GetTokenType (this Token tok) {
- return (EbnfTokenType)tok.Type;
- }
- public static void SetTokenType (this Token tok, EbnfTokenType type) {
- tok.Type = (TokenType)type;
- }
- }
public class EbnfDocument : SourceDocument {
public EbnfDocument (string fullPath, string editorPath) : base (fullPath, editorPath) {
EbnfTokenType xmlTokType = (EbnfTokenType)tokType;
if (xmlTokType == EbnfTokenType.OpenBracket || xmlTokType == EbnfTokenType.ClosingBracket)
return Colors.RebeccaPurple;
- if (xmlTokType == EbnfTokenType.StringDelimiter)
+ if (xmlTokType == EbnfTokenType.DoubleQuote)
return Colors.Teal;
if (xmlTokType == EbnfTokenType.StringLiteral)
return Colors.Teal;
namespace CrowEdit.Ebnf
{
+ public static class Extensions {
+ public static EbnfTokenType GetTokenType (this Token tok) {
+ return (EbnfTokenType)tok.Type;
+ }
+ public static void SetTokenType (this Token tok, EbnfTokenType type) {
+ tok.Type = (TokenType)type;
+ }
+ public static bool Is(this Token tok, EbnfTokenType type) => (EbnfTokenType)tok.Type == type;
+ }
+
public class EbnfSyntaxAnalyser : SyntaxAnalyser {
public EbnfSyntaxAnalyser (EbnfDocument document) : base (document) {}
-
- // ::= NCName '::=' Expression
+
+ bool skipTriviaAndComments(MultiNodeSyntax currentNode) {
+ while (tryPeekFlag(out Token token, TokenType.Trivia)) {
+ switch(token.GetTokenType()) {
+ case (EbnfTokenType)TokenType.LineBreak:
+ Read();
+ break;
+ /*case EbnfTokenType.LineCommentStart:
+ MultiNodeSyntax cmt = new CommentTriviaSyntax(false);
+ cmt.AddChild(new SingleTokenSyntax(Read()));
+ if (tryPeek(TokenType.LineComment))
+ cmt.AddChild(new SingleTokenSyntax(Read()));
+ currentNode.AddChild(cmt);
+ break;*/
+ case EbnfTokenType.BlockCommentStart:
+ MultiNodeSyntax bc = new CommentTriviaSyntax(true);
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ while(tryPeek(out Token tok)) {
+ if (tok.Type == TokenType.BlockCommentEnd) {
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ break;
+ }
+ if (tok.Type == TokenType.LineBreak)
+ Read();
+ else
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ }
+ currentNode.AddChild(bc);
+ break;
+ default:
+ Read();
+ break;
+ }
+ }
+ return !EOF;
+ }
+ bool accept(MultiNodeSyntax node, Enum tokenType) {
+ if (EOF)
+ return false;
+ if (Peek().Type == (TokenType)tokenType) {
+ node.AddChild(new SingleTokenSyntax(Read()));
+ return true;
+ }
+ return false;
+ }
+ // Production ::= NCName '::=' Expression
+ ProductionSyntax processNode(ProductionSyntax prod) {
+ if (accept(prod, EbnfTokenType.SymbolName))
+ if (skipTriviaAndComments(prod))
+ if (accept(prod, EbnfTokenType.DefiningSymbol))
+ prod.AddChild(processNode(new ExpressionSyntax()));
+ return prod;
+ }
+ // Expression ::= ( Choice | Link )
+ ExpressionSyntax processNode(ExpressionSyntax exp) {
+ skipTriviaAndComments(exp);
+ while(!EOF) {
+ if (cancel.IsCancellationRequested)
+ break;
+
+ }
+
+ if (accept(exp, EbnfTokenType.SymbolName))
+ if (skipTriviaAndComments(exp))
+ if (accept(exp, EbnfTokenType.DefiningSymbol))
+ exp.AddChild(processNode(new ExpressionSyntax()));
+ return exp;
+ }
// Link ::= '[' URL ']'
// Choice ::= SequenceOrDifference ( '|' SequenceOrDifference )*
+
// SequenceOrDifference ::= Item ( '-' Item | Item* )?
// Item ::= Primary ( '?' | '*' | '+' )?
- //NCName | StringLiteral | CharCode | CharClass | '(' Choice ')'
+
+ // Primary ::= NCName | StringLiteral | CharCode | CharClass | '(' Choice ')'
// StringLiteral ::= '"' [^"]* '"' | "'" [^']* "'"
public override async Task<SyntaxRootNode> Process(CancellationToken cancel = default)
{
Tokenizer tokenizer = new EbnfTokenizer();
ReadOnlyTextBuffer buff = document.ImmutableBufferCopy;
- Token[] tokens = tokenizer.Tokenize(buff.Source.Span);
- Root = new EbnfRootSyntax (buff, tokens);
+ Token[] tokens = tokenizer.Tokenize(buff.Source.Span);
- currentLine = 0;
tokIdx = 0;
-
+ this.cancel = cancel;
+
+ Root = new EbnfRootSyntax (buff, tokens);
+
+
+ /*while (!EOF) {
+ if (cancel.IsCancellationRequested)
+ break;
+ if (!skipTriviaAndComments(Root))
+ break;
+ if (!Peek().Is(EbnfTokenType.SymbolName)) {
+ Root.AddChild(new UnexpectedTokenSyntax(Read()));
+ continue;
+ }
+ Root.AddChild(processNode(new ProductionSyntax()));
+ } */
/*while(tokIdx < tokens.Length) {
skipTrivia();
bool EndOfExpression =>
- EOF || tokIdx > tokens.Length - 2 || tokens[tokIdx + 1].GetTokenType() == EbnfTokenType.SymbolAffectation;
+ EOF || tokIdx > tokens.Length - 2 || tokens[tokIdx + 1].GetTokenType() == EbnfTokenType.DefiningSymbol;
bool resolvStackPeekIsOpenBracket =>
- resolveStack.TryPeek (out object elt) && elt is Token tok && tok.GetTokenType() == EbnfTokenType.OpenRoundBracket;
+ resolveStack.TryPeek (out object elt) && elt is Token tok && tok.GetTokenType() == EbnfTokenType.OpenBrace;
bool resolvStackPeekIsSequenceOperator =>
resolveStack.TryPeek (out object elt) && elt is Expression;
if (resolveStack.TryPeek (out object obj)) {
if (obj is Token tok) {
- if (tok.GetTokenType() == EbnfTokenType.OpenRoundBracket)
+ if (tok.GetTokenType() == EbnfTokenType.OpenBrace)
return rightOp;
resolveStack.Pop ();
resolveStack.Push (resolve (leftOp));
else
resolveStack.Push (leftOp);
- } else if (tok.GetTokenType() == EbnfTokenType.OpenRoundBracket)
+ } else if (tok.GetTokenType() == EbnfTokenType.OpenBrace)
resolveStack.Push (leftOp);
} else //so theres an expression on the stack, the operator is sequenceOp (whitespace) with precedence = 3
resolveStack.Push (resolve (leftOp));
if (Peek.GetTokenType() != EbnfTokenType.SymbolName)
throw new EbnfParserException ($"expecing symbol name, having {Peek.GetTokenType()}, {Peek.AsString (source2)}");
curSymbol = new SymbolDecl (Read ().AsString (source2));
- if (!tryRead (out tok, EbnfTokenType.SymbolAffectation))
+ if (!tryRead (out tok, EbnfTokenType.DefiningSymbol))
throw new EbnfParserException ($"expecing '::='");
resolveStack = new Stack<object> (16);
- } else if (Peek.GetTokenType() == EbnfTokenType.OpenRoundBracket) {
+ } else if (Peek.GetTokenType() == EbnfTokenType.OpenBrace) {
tok = Read ();
resolveStack.Push (tok);
- } else if (Peek.GetTokenType() == EbnfTokenType.ClosingRoundBracket) {
+ } else if (Peek.GetTokenType() == EbnfTokenType.ClosingBrace) {
tok = Read ();
Expression rightOp = resolve ();
while (!resolvStackPeekIsOpenBracket)
rightOp = resolve (rightOp);
- if (resolveStack.TryPop (out object obj) && obj is Token tk && tk.GetTokenType() == EbnfTokenType.OpenRoundBracket)
+ if (resolveStack.TryPop (out object obj) && obj is Token tk && tk.GetTokenType() == EbnfTokenType.OpenBrace)
checkCardinalityAndPushNewExpression (rightOp);
else
throw new EbnfParserException ($"expecing open bracket.");
throw new EbnfParserException ($"malformed character range match");
}
}
- if (tok.GetTokenType() == EbnfTokenType.StringDelimiter) {
+ if (tok.GetTokenType() == EbnfTokenType.DoubleQuote) {
if (tryRead (out tok, EbnfTokenType.StringMatch)) {
te = new StringMatch (tok.AsString (source2));
if (!tryRead (out tok, EbnfTokenType.StringLiteral))
checkCardinalityAndPushNewExpression (new SymbolMatch (tok.AsString (source2)));
} else if (Peek.GetTokenType().HasFlag (EbnfTokenType.Operator)) {
Token newOp = Read ();
- if (newOp.GetTokenType() == EbnfTokenType.SymbolAffectation || newOp.GetTokenType() == EbnfTokenType.CardinalityOp)
+ if (newOp.GetTokenType() == EbnfTokenType.DefiningSymbol || newOp.GetTokenType() == EbnfTokenType.CardinalityOp)
System.Diagnostics.Debugger.Break ();
if (resolveStackTryPeek<Expression> (out Expression exp)) {
resolveStack.Push (resolve (exp));
else
resolveStack.Push (exp);
- } else if (tok.GetTokenType() == EbnfTokenType.OpenRoundBracket)
+ } else if (tok.GetTokenType() == EbnfTokenType.OpenBrace)
resolveStack.Push (exp);
} else if (3 <= operatorPrecedance (newOp)) { //so theres an expression on the stack, the operator is sequenceOp (whitespace) with precedence = 3
resolveStack.Push (resolve (exp));
public class EbnfSyntaxNode : SyntaxNode {
}
- public class ProductionSyntax : SyntaxNode {
+ public class ProductionSyntax : MultiNodeSyntax {
}
- public class ExpressionSyntax : SyntaxNode {
+ public class ExpressionSyntax : MultiNodeSyntax {
}
public class LinkSyntax : ExpressionSyntax {
}
- public class ChoiceSyntax : ExpressionSyntax {
+ public class ChoiceSyntax : MultiNodeSyntax {
}
// (Item ( '-' Item | Item* ))?
- public class SequenceOrDifferenceSyntax : SyntaxNode {
+ public class SequenceOrDifferenceSyntax : MultiNodeSyntax {
}
// Item ::= Primary ( '?' | '*' | '+' )? */
- public class ItemSyntax : SyntaxNode {
+ public class ItemSyntax : MultiNodeSyntax {
}
/* NCName | StringLiteral | CharCode | CharClass | '(' Choice ')' */
- public class PrimarySyntax : SyntaxNode {
- }
+ public class PrimarySyntax : SyntaxNode {}
+
// StringLiteral ::= '"' [^"]* '"' | "'" [^']* "'"
public class StringLiteralSyntax : SyntaxNode {
}
EndOfFile = 0x4103,
LineComment = 0x0103,
BlockCommentStart = 0x0104,
- BlockComment = 0x0105,
+ BlockCommentPart = 0x0105,
BlockCommentEnd = 0x0106,
+
Name = 0x0200,
SymbolName = 0x0201,
Punctuation = 0x0400,
- OpenRoundBracket = 0x0401,
- ClosingRoundBracket = 0x0402,
+ OpenBrace = 0x0401,// '('
+ ClosingBrace = 0x0402,// ')'
OpenBracket = 0x0403,// '['
ClosingBracket = 0x0404,// ']'
- StringDelimiter = 0x0405,
+ DoubleQuote = 0x0405,
+ SingleQuote = 0x0405,
+
StringLiteral = 0x0406,
CharMatchNegation = 0x0407,// '^'
CharMatchRangeOperator = 0x0C01,// '-'
//CharMatch = 0x0C01,// '-'
Operator = 0x0800,
- SymbolAffectation = 0x0801,
+ DefiningSymbol = 0x0801,
ChoiceOp = 0x0802,
ExclusionOp = 0x0803,
SequenceOp = 0x0804, //never parsed (it's only a white space) but use in syntaxAnalyser
addTok (ref reader, EbnfTokenType.BlockCommentStart);
while (!reader.EndOfSpan) {
if (reader.Eol()) {
- addTok (ref reader, EbnfTokenType.BlockComment);
+ addTok (ref reader, EbnfTokenType.BlockCommentPart);
reader.ReadEol();
addTok (ref reader, EbnfTokenType.LineBreak);
continue;
}
if (reader.TryPeek ("*/")) {
- addTok (ref reader, EbnfTokenType.BlockComment);
+ addTok (ref reader, EbnfTokenType.BlockCommentPart);
reader.Advance (2);
addTok (ref reader, EbnfTokenType.BlockCommentEnd);
break;
case '"':
case '\'':
char q = reader.Read();
- addTok (ref reader, EbnfTokenType.StringDelimiter);
+ addTok (ref reader, EbnfTokenType.DoubleQuote);
while (!reader.EndOfSpan) {
if (reader.Eol()) {
addTok (ref reader, EbnfTokenType.StringLiteral);
} else if (reader.Peek == q) {
addTok (ref reader, EbnfTokenType.StringLiteral);
reader.Advance ();
- addTok (ref reader, EbnfTokenType.StringDelimiter);
+ addTok (ref reader, EbnfTokenType.DoubleQuote);
break;
}
reader.Advance();
reader.Advance();
if (!reader.TryRead (":="))
throw new EbnfParserException ("malform symbol declaration, expecting '::='.");
- addTok (ref reader, EbnfTokenType.SymbolAffectation);
+ addTok (ref reader, EbnfTokenType.DefiningSymbol);
break;
case '<':
reader.Advance();
break;
case '(':
reader.Advance();
- addTok (ref reader, EbnfTokenType.OpenRoundBracket);
+ addTok (ref reader, EbnfTokenType.OpenBrace);
break;
case ')':
reader.Advance();
- addTok (ref reader, EbnfTokenType.ClosingRoundBracket);
+ addTok (ref reader, EbnfTokenType.ClosingBrace);
break;
case '|':
reader.Advance();
-// Copyright (c) 2021-2021 Bruyère Jean-Philippe <jp_bruyere@hotmail.com>
+// Copyright (c) 2021-2025 Bruyère Jean-Philippe <jp_bruyere@hotmail.com>
//
// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT)
using System;
namespace CrowEdit.Xml
{
- public static class Extensions {
- public static XmlTokenType GetTokenType (this Token tok) {
- return (XmlTokenType)tok.Type;
- }
- public static void SetTokenType (this Token tok, XmlTokenType type) {
- tok.Type = (TokenType)type;
- }
- public static bool Is(this Token tok, XmlTokenType type) => (XmlTokenType)tok.Type == type;
- }
public class XmlDocument : SourceDocument {
public XmlDocument (string fullPath, string editorPath) : base (fullPath, editorPath) { }
-// Copyright (c) 2021-2021 Bruyère Jean-Philippe <jp_bruyere@hotmail.com>
+// Copyright (c) 2021-2025 Bruyère Jean-Philippe <jp_bruyere@hotmail.com>
//
// This code is licensed under the MIT license (MIT) (http://opensource.org/licenses/MIT)
using System;
namespace CrowEdit.Xml
{
+ public static class Extensions {
+ public static XmlTokenType GetTokenType (this Token tok) {
+ return (XmlTokenType)tok.Type;
+ }
+ public static void SetTokenType (this Token tok, XmlTokenType type) {
+ tok.Type = (TokenType)type;
+ }
+ public static bool Is(this Token tok, XmlTokenType type) => (XmlTokenType)tok.Type == type;
+ }
public class XmlSyntaxAnalyser : SyntaxAnalyser {
public XmlSyntaxAnalyser (XmlDocument document) : base (document) {}
+ bool skipTriviaAndComments(MultiNodeSyntax currentNode, bool skipLineBreaks = true) {
+ while (tryPeekFlag(out Token token, TokenType.Trivia)) {
+ switch(token.GetTokenType()) {
+ case (XmlTokenType)TokenType.LineBreak:
+ if (!skipLineBreaks)
+ return true;
+ Read();
+ break;
+ case XmlTokenType.BlockCommentStart:
+ MultiNodeSyntax bc = new CommentTriviaSyntax(true);
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ while(tryPeek(out Token tok)) {
+ if (tok.Type == TokenType.BlockCommentEnd) {
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ break;
+ }
+ if (tok.Type == TokenType.LineBreak) {
+ if (!skipLineBreaks)
+ return true;
+ Read();
+ } else {
+ bc.AddChild(new SingleTokenSyntax(Read()));
+ }
+ }
+ currentNode.AddChild(bc);
+ break;
+ default:
+ Read();
+ break;
+ }
+ }
+
+ return !EOF;
+ }
+ bool accept(MultiNodeSyntax node, Enum tokenType) {
+ if (EOF)
+ return false;
+ if (Peek().Type == (TokenType)tokenType) {
+ node.AddChild(new SingleTokenSyntax(Read()));
+ return true;
+ }
+ return false;
+ }
public virtual void ProcessAttributeValueSyntax(AttributeSyntax attrib) {
//attrib.valueTok = tokIdx - attrib.TokenIndexBase;
}
+ AttributeSyntax processNode(AttributeSyntax attrib) {
+ if (accept(attrib, XmlTokenType.EqualSign))
+ if (accept(attrib, XmlTokenType.AttributeValueOpen))
+ if(accept(attrib, XmlTokenType.AttributeValue))
+ accept(attrib, XmlTokenType.AttributeValueClose);
+ return attrib;
+ }
+ ElementEndTagSyntax processNode(ElementEndTagSyntax et) {
+ if (accept(et, XmlTokenType.ElementName))
+ accept(et, XmlTokenType.ClosingSign);
+ return et;
+ }
+ ProcessingInstructionSyntax processNode(ProcessingInstructionSyntax pi) {
+ if (Peek().Is(XmlTokenType.PI_Target)) {
+ pi.AddChild(new PITargetSyntax(Read()));
+ while (skipTriviaAndComments(pi, false)) {
+ if (Peek().Is(XmlTokenType.PI_End)) {
+ pi.AddChild(new SingleTokenSyntax(Read()));
+ break;
+ }
+ if (Peek().Is(XmlTokenType.AttributeName))
+ pi.AddChild(processNode(new AttributeSyntax(Read())));
+ else
+ pi.AddChild(new UnexpectedTokenSyntax(Read()));
+ }
+ }
+ return pi;
+ }
+
+ void processElementNode(MultiNodeSyntax node) {
+ ElementStartTagSyntax start = new ElementStartTagSyntax(Read());
+ if (accept (start, XmlTokenType.ElementName)) {
+ while (skipTriviaAndComments(node)) {
+ if (accept (start, XmlTokenType.EmptyElementClosing)) {
+ node.AddChild(new EmptyElementSyntax(start));
+ break;
+ }
+ if (accept (start, XmlTokenType.ClosingSign)) {
+ node.AddChild(processElement(new ElementSyntax(start)));
+ break;
+ }
+ if (Peek().Is(XmlTokenType.AttributeName))
+ start.AddChild(processNode(new AttributeSyntax(Read())));
+ else
+ start.AddChild(new UnexpectedTokenSyntax(Read()));
+ }
+ } else {
+ start.AddChild(new UnexpectedTokenSyntax(Read()));
+ node.AddChild(new ElementSyntax(start));
+ }
+ }
+ ElementSyntax processElement(ElementSyntax elt) {
+ while (!EOF) {
+ if (cancel.IsCancellationRequested)
+ break;
+ if (!skipTriviaAndComments(elt))
+ break;
+ if (Peek().Is(XmlTokenType.ElementOpen)) {
+ processElementNode(elt);
+ } else if (Peek().Is(XmlTokenType.EndElementOpen)) {
+ elt.AddChild(processNode(new ElementEndTagSyntax(Read())));
+ break;
+ } else if (Peek().Is(XmlTokenType.PI_Start)) {
+ elt.AddChild(processNode(new ProcessingInstructionSyntax(Read())));
+ } else {
+ elt.AddChild(new UnexpectedTokenSyntax(Read()));
+ }
+ }
+ return elt;
+ }
public override async Task<SyntaxRootNode> Process (CancellationToken cancel = default) {
Tokenizer tokenizer = new XmlTokenizer();
ReadOnlyTextBuffer buff = document.ImmutableBufferCopy;
Token[] tokens = tokenizer.Tokenize(buff.Source.Span);
- Root = new XMLRootSyntax (buff, tokens);
-
- currentLine = 0;
tokIdx = 0;
-
+ this.cancel = cancel;//?
+
+ Root = new XMLRootSyntax (buff, tokens);
+ while (!EOF) {
+ if (cancel.IsCancellationRequested)
+ break;
+ if (!skipTriviaAndComments(Root))
+ break;
+ if (Peek().Is(XmlTokenType.ElementOpen)) {
+ processElementNode(Root);
+ } else if (Peek().Is(XmlTokenType.PI_Start)) {
+ Root.AddChild(processNode(new ProcessingInstructionSyntax(Read())));
+ } else {
+ Root.AddChild(new UnexpectedTokenSyntax(Read()));
+ }
+ }
/*while (tokIdx < tokens.Length) {
if (curTok.Type == TokenType.LineBreak)
currentLine++;
}
public class ProcessingInstructionSyntax : MultiNodeSyntax {
// public override bool IsComplete => base.IsComplete & name.HasValue & PIClose.HasValue;
- public ProcessingInstructionSyntax (){}
+ public ProcessingInstructionSyntax (Token openTok){
+ AddChild(new SingleTokenSyntax(openTok));
+ }
+ }
+ public class PITargetSyntax : SingleTokenSyntax {
+// public override bool IsComplete => base.IsComplete & name.HasValue & PIClose.HasValue;
+ public PITargetSyntax (Token target) : base(target) { }
}
- public abstract class ElementTagSyntax : SyntaxNode {
+
+ public abstract class ElementTagSyntax : MultiNodeSyntax {
// public override bool IsComplete => base.IsComplete & name.HasValue & close.HasValue;
- protected ElementTagSyntax () { }
+ protected ElementTagSyntax (Token openTok) {
+ AddChild(new SingleTokenSyntax(openTok));
+ }
}
+ /*public class ElementNameSyntax : SingleTokenSyntax {
+ public ElementNameSyntax(Token name) : base(name) {}
+ }*/
public class ElementStartTagSyntax : ElementTagSyntax {
- public ElementStartTagSyntax () {}
+ public ElementStartTagSyntax (Token openTok) : base(openTok) {}
}
public class ElementEndTagSyntax : ElementTagSyntax {
- public ElementEndTagSyntax () { }
+ public ElementEndTagSyntax (Token openTok) : base(openTok) {}
}
public class EmptyElementSyntax : MultiNodeSyntax {
public class ElementSyntax : MultiNodeSyntax {
- //public override bool IsComplete => base.IsComplete & StartTag.IsComplete & (EndTag != null && EndTag.IsComplete);
+ public override bool IsComplete => base.IsComplete;// & StartTag.IsComplete & (EndTag != null && EndTag.IsComplete);
- public ElementSyntax (ElementStartTagSyntax startTag) {
- AddChild (startTag);
+ public ElementSyntax (ElementStartTagSyntax startNode) {
+ AddChild (startNode);
}
}
-
public class AttributeSyntax : MultiNodeSyntax {
//public override bool IsComplete => base.IsComplete & name.HasValue & equal.HasValue & valueTok.HasValue & valueOpen.HasValue & valueClose.HasValue;
+ public AttributeSyntax(Token name) {
+ AddChild (new SingleTokenSyntax(name));
+ }
}
+ /*public class AttributeNameSyntax : SingleTokenSyntax {
+ public AttributeNameSyntax(Token name) : base(name) {}
+ }*/
}
\ No newline at end of file
home / projects / jp / crowedit.git / commitdiff