Add automatic graph generation tool

tools/grammar-analysis/LexBasic.py

@@ -0,0 +1,175 @@
+# Generated from java-escape by ANTLR 4.5
+from antlr4 import *
+from io import StringIO
+
+
+def serializedATN():
+    with StringIO() as buf:
+        buf.write("\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2\2")
+        buf.write("\u0130\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7")
+        buf.write("\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r")
+        buf.write("\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22\4\23")
+        buf.write("\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30")
+        buf.write("\4\31\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36")
+        buf.write("\t\36\4\37\t\37\4 \t \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%")
+        buf.write("\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t+\4,\t,\4-\t-\4.")
+        buf.write("\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\3\2\3\2\5\2h\n")
+        buf.write("\2\3\3\3\3\3\4\3\4\3\5\3\5\3\5\3\5\7\5r\n\5\f\5\16\5u")
+        buf.write("\13\5\3\5\3\5\3\5\5\5z\n\5\3\6\3\6\3\6\3\6\3\6\7\6\u0081")
+        buf.write("\n\6\f\6\16\6\u0084\13\6\3\6\3\6\3\6\5\6\u0089\n\6\3\7")
+        buf.write("\3\7\3\7\3\7\7\7\u008f\n\7\f\7\16\7\u0092\13\7\3\b\3\b")
+        buf.write("\3\b\3\b\3\b\5\b\u0099\n\b\3\t\3\t\3\t\3\n\3\n\3\n\3\n")
+        buf.write("\3\n\5\n\u00a3\n\n\5\n\u00a5\n\n\5\n\u00a7\n\n\5\n\u00a9")
+        buf.write("\n\n\3\13\3\13\3\13\7\13\u00ae\n\13\f\13\16\13\u00b1\13")
+        buf.write("\13\5\13\u00b3\n\13\3\f\3\f\3\r\3\r\3\16\3\16\3\16\3\16")
+        buf.write("\3\16\3\16\3\16\3\16\3\16\5\16\u00c2\n\16\3\17\3\17\3")
+        buf.write("\17\5\17\u00c7\n\17\3\17\3\17\3\20\3\20\3\20\7\20\u00ce")
+        buf.write("\n\20\f\20\16\20\u00d1\13\20\3\20\3\20\3\21\3\21\3\21")
+        buf.write("\7\21\u00d8\n\21\f\21\16\21\u00db\13\21\3\21\3\21\3\22")
+        buf.write("\3\22\3\22\7\22\u00e2\n\22\f\22\16\22\u00e5\13\22\3\23")
+        buf.write("\3\23\3\23\3\23\5\23\u00eb\n\23\3\24\3\24\3\25\3\25\3")
+        buf.write("\25\3\25\3\26\3\26\3\27\3\27\3\30\3\30\3\30\3\31\3\31")
+        buf.write("\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37")
+        buf.write("\3\37\3 \3 \3!\3!\3!\3\"\3\"\3#\3#\3$\3$\3%\3%\3&\3&\3")
+        buf.write("\'\3\'\3(\3(\3(\3)\3)\3*\3*\3+\3+\3,\3,\3-\3-\3.\3.\3")
+        buf.write("/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\4s\u0082\2\63\3")
+        buf.write("\2\5\2\7\2\t\2\13\2\r\2\17\2\21\2\23\2\25\2\27\2\31\2")
+        buf.write("\33\2\35\2\37\2!\2#\2%\2\'\2)\2+\2-\2/\2\61\2\63\2\65")
+        buf.write("\2\67\29\2;\2=\2?\2A\2C\2E\2G\2I\2K\2M\2O\2Q\2S\2U\2W")
+        buf.write("\2Y\2[\2]\2_\2a\2c\2\3\2\r\4\2\13\13\"\"\4\2\f\f\16\17")
+        buf.write("\4\2\f\f\17\17\n\2$$))^^ddhhppttvv\3\2\63;\5\2\62;CHc")
+        buf.write("h\3\2\62;\6\2\f\f\17\17))^^\6\2\f\f\17\17$$^^\5\2\u00b9")
+        buf.write("\u00b9\u0302\u0371\u2041\u2042\17\2C\\c|\u00c2\u00d8\u00da")
+        buf.write("\u00f8\u00fa\u0301\u0372\u037f\u0381\u2001\u200e\u200f")
+        buf.write("\u2072\u2191\u2c02\u2ff1\u3003\ud801\uf902\ufdd1\ufdf2")
+        buf.write("\uffff\u0118\3g\3\2\2\2\5i\3\2\2\2\7k\3\2\2\2\tm\3\2\2")
+        buf.write("\2\13{\3\2\2\2\r\u008a\3\2\2\2\17\u0093\3\2\2\2\21\u009a")
+        buf.write("\3\2\2\2\23\u009d\3\2\2\2\25\u00b2\3\2\2\2\27\u00b4\3")
+        buf.write("\2\2\2\31\u00b6\3\2\2\2\33\u00c1\3\2\2\2\35\u00c3\3\2")
+        buf.write("\2\2\37\u00ca\3\2\2\2!\u00d4\3\2\2\2#\u00de\3\2\2\2%\u00ea")
+        buf.write("\3\2\2\2\'\u00ec\3\2\2\2)\u00ee\3\2\2\2+\u00f2\3\2\2\2")
+        buf.write("-\u00f4\3\2\2\2/\u00f6\3\2\2\2\61\u00f9\3\2\2\2\63\u00fb")
+        buf.write("\3\2\2\2\65\u00fd\3\2\2\2\67\u00ff\3\2\2\29\u0101\3\2")
+        buf.write("\2\2;\u0103\3\2\2\2=\u0105\3\2\2\2?\u0107\3\2\2\2A\u0109")
+        buf.write("\3\2\2\2C\u010c\3\2\2\2E\u010e\3\2\2\2G\u0110\3\2\2\2")
+        buf.write("I\u0112\3\2\2\2K\u0114\3\2\2\2M\u0116\3\2\2\2O\u0118\3")
+        buf.write("\2\2\2Q\u011b\3\2\2\2S\u011d\3\2\2\2U\u011f\3\2\2\2W\u0121")
+        buf.write("\3\2\2\2Y\u0123\3\2\2\2[\u0125\3\2\2\2]\u0127\3\2\2\2")
+        buf.write("_\u012a\3\2\2\2a\u012c\3\2\2\2c\u012e\3\2\2\2eh\5\5\3")
+        buf.write("\2fh\5\7\4\2ge\3\2\2\2gf\3\2\2\2h\4\3\2\2\2ij\t\2\2\2")
+        buf.write("j\6\3\2\2\2kl\t\3\2\2l\b\3\2\2\2mn\7\61\2\2no\7,\2\2o")
+        buf.write("s\3\2\2\2pr\13\2\2\2qp\3\2\2\2ru\3\2\2\2st\3\2\2\2sq\3")
+        buf.write("\2\2\2ty\3\2\2\2us\3\2\2\2vw\7,\2\2wz\7\61\2\2xz\7\2\2")
+        buf.write("\3yv\3\2\2\2yx\3\2\2\2z\n\3\2\2\2{|\7\61\2\2|}\7,\2\2")
+        buf.write("}~\7,\2\2~\u0082\3\2\2\2\177\u0081\13\2\2\2\u0080\177")
+        buf.write("\3\2\2\2\u0081\u0084\3\2\2\2\u0082\u0083\3\2\2\2\u0082")
+        buf.write("\u0080\3\2\2\2\u0083\u0088\3\2\2\2\u0084\u0082\3\2\2\2")
+        buf.write("\u0085\u0086\7,\2\2\u0086\u0089\7\61\2\2\u0087\u0089\7")
+        buf.write("\2\2\3\u0088\u0085\3\2\2\2\u0088\u0087\3\2\2\2\u0089\f")
+        buf.write("\3\2\2\2\u008a\u008b\7\61\2\2\u008b\u008c\7\61\2\2\u008c")
+        buf.write("\u0090\3\2\2\2\u008d\u008f\n\4\2\2\u008e\u008d\3\2\2\2")
+        buf.write("\u008f\u0092\3\2\2\2\u0090\u008e\3\2\2\2\u0090\u0091\3")
+        buf.write("\2\2\2\u0091\16\3\2\2\2\u0092\u0090\3\2\2\2\u0093\u0098")
+        buf.write("\5+\26\2\u0094\u0099\t\5\2\2\u0095\u0099\5\23\n\2\u0096")
+        buf.write("\u0099\13\2\2\2\u0097\u0099\7\2\2\3\u0098\u0094\3\2\2")
+        buf.write("\2\u0098\u0095\3\2\2\2\u0098\u0096\3\2\2\2\u0098\u0097")
+        buf.write("\3\2\2\2\u0099\20\3\2\2\2\u009a\u009b\5+\26\2\u009b\u009c")
+        buf.write("\13\2\2\2\u009c\22\3\2\2\2\u009d\u00a8\7w\2\2\u009e\u00a6")
+        buf.write("\5\27\f\2\u009f\u00a4\5\27\f\2\u00a0\u00a2\5\27\f\2\u00a1")
+        buf.write("\u00a3\5\27\f\2\u00a2\u00a1\3\2\2\2\u00a2\u00a3\3\2\2")
+        buf.write("\2\u00a3\u00a5\3\2\2\2\u00a4\u00a0\3\2\2\2\u00a4\u00a5")
+        buf.write("\3\2\2\2\u00a5\u00a7\3\2\2\2\u00a6\u009f\3\2\2\2\u00a6")
+        buf.write("\u00a7\3\2\2\2\u00a7\u00a9\3\2\2\2\u00a8\u009e\3\2\2\2")
+        buf.write("\u00a8\u00a9\3\2\2\2\u00a9\24\3\2\2\2\u00aa\u00b3\7\62")
+        buf.write("\2\2\u00ab\u00af\t\6\2\2\u00ac\u00ae\5\31\r\2\u00ad\u00ac")
+        buf.write("\3\2\2\2\u00ae\u00b1\3\2\2\2\u00af\u00ad\3\2\2\2\u00af")
+        buf.write("\u00b0\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00af\3\2\2\2")
+        buf.write("\u00b2\u00aa\3\2\2\2\u00b2\u00ab\3\2\2\2\u00b3\26\3\2")
+        buf.write("\2\2\u00b4\u00b5\t\7\2\2\u00b5\30\3\2\2\2\u00b6\u00b7")
+        buf.write("\t\b\2\2\u00b7\32\3\2\2\2\u00b8\u00b9\7v\2\2\u00b9\u00ba")
+        buf.write("\7t\2\2\u00ba\u00bb\7w\2\2\u00bb\u00c2\7g\2\2\u00bc\u00bd")
+        buf.write("\7h\2\2\u00bd\u00be\7c\2\2\u00be\u00bf\7n\2\2\u00bf\u00c0")
+        buf.write("\7u\2\2\u00c0\u00c2\7g\2\2\u00c1\u00b8\3\2\2\2\u00c1\u00bc")
+        buf.write("\3\2\2\2\u00c2\34\3\2\2\2\u00c3\u00c6\5\61\31\2\u00c4")
+        buf.write("\u00c7\5\17\b\2\u00c5\u00c7\n\t\2\2\u00c6\u00c4\3\2\2")
+        buf.write("\2\u00c6\u00c5\3\2\2\2\u00c7\u00c8\3\2\2\2\u00c8\u00c9")
+        buf.write("\5\61\31\2\u00c9\36\3\2\2\2\u00ca\u00cf\5\61\31\2\u00cb")
+        buf.write("\u00ce\5\17\b\2\u00cc\u00ce\n\t\2\2\u00cd\u00cb\3\2\2")
+        buf.write("\2\u00cd\u00cc\3\2\2\2\u00ce\u00d1\3\2\2\2\u00cf\u00cd")
+        buf.write("\3\2\2\2\u00cf\u00d0\3\2\2\2\u00d0\u00d2\3\2\2\2\u00d1")
+        buf.write("\u00cf\3\2\2\2\u00d2\u00d3\5\61\31\2\u00d3 \3\2\2\2\u00d4")
+        buf.write("\u00d9\5\63\32\2\u00d5\u00d8\5\17\b\2\u00d6\u00d8\n\n")
+        buf.write("\2\2\u00d7\u00d5\3\2\2\2\u00d7\u00d6\3\2\2\2\u00d8\u00db")
+        buf.write("\3\2\2\2\u00d9\u00d7\3\2\2\2\u00d9\u00da\3\2\2\2\u00da")
+        buf.write("\u00dc\3\2\2\2\u00db\u00d9\3\2\2\2\u00dc\u00dd\5\63\32")
+        buf.write("\2\u00dd\"\3\2\2\2\u00de\u00e3\5\61\31\2\u00df\u00e2\5")
+        buf.write("\17\b\2\u00e0\u00e2\n\t\2\2\u00e1\u00df\3\2\2\2\u00e1")
+        buf.write("\u00e0\3\2\2\2\u00e2\u00e5\3\2\2\2\u00e3\u00e1\3\2\2\2")
+        buf.write("\u00e3\u00e4\3\2\2\2\u00e4$\3\2\2\2\u00e5\u00e3\3\2\2")
+        buf.write("\2\u00e6\u00eb\5\'\24\2\u00e7\u00eb\4\62;\2\u00e8\u00eb")
+        buf.write("\5Q)\2\u00e9\u00eb\t\13\2\2\u00ea\u00e6\3\2\2\2\u00ea")
+        buf.write("\u00e7\3\2\2\2\u00ea\u00e8\3\2\2\2\u00ea\u00e9\3\2\2\2")
+        buf.write("\u00eb&\3\2\2\2\u00ec\u00ed\t\f\2\2\u00ed(\3\2\2\2\u00ee")
+        buf.write("\u00ef\7k\2\2\u00ef\u00f0\7p\2\2\u00f0\u00f1\7v\2\2\u00f1")
+        buf.write("*\3\2\2\2\u00f2\u00f3\7^\2\2\u00f3,\3\2\2\2\u00f4\u00f5")
+        buf.write("\7<\2\2\u00f5.\3\2\2\2\u00f6\u00f7\7<\2\2\u00f7\u00f8")
+        buf.write("\7<\2\2\u00f8\60\3\2\2\2\u00f9\u00fa\7)\2\2\u00fa\62\3")
+        buf.write("\2\2\2\u00fb\u00fc\7$\2\2\u00fc\64\3\2\2\2\u00fd\u00fe")
+        buf.write("\7*\2\2\u00fe\66\3\2\2\2\u00ff\u0100\7+\2\2\u01008\3\2")
+        buf.write("\2\2\u0101\u0102\7}\2\2\u0102:\3\2\2\2\u0103\u0104\7\177")
+        buf.write("\2\2\u0104<\3\2\2\2\u0105\u0106\7]\2\2\u0106>\3\2\2\2")
+        buf.write("\u0107\u0108\7_\2\2\u0108@\3\2\2\2\u0109\u010a\7/\2\2")
+        buf.write("\u010a\u010b\7@\2\2\u010bB\3\2\2\2\u010c\u010d\7>\2\2")
+        buf.write("\u010dD\3\2\2\2\u010e\u010f\7@\2\2\u010fF\3\2\2\2\u0110")
+        buf.write("\u0111\7?\2\2\u0111H\3\2\2\2\u0112\u0113\7A\2\2\u0113")
+        buf.write("J\3\2\2\2\u0114\u0115\7,\2\2\u0115L\3\2\2\2\u0116\u0117")
+        buf.write("\7-\2\2\u0117N\3\2\2\2\u0118\u0119\7-\2\2\u0119\u011a")
+        buf.write("\7?\2\2\u011aP\3\2\2\2\u011b\u011c\7a\2\2\u011cR\3\2\2")
+        buf.write("\2\u011d\u011e\7~\2\2\u011eT\3\2\2\2\u011f\u0120\7&\2")
+        buf.write("\2\u0120V\3\2\2\2\u0121\u0122\7.\2\2\u0122X\3\2\2\2\u0123")
+        buf.write("\u0124\7=\2\2\u0124Z\3\2\2\2\u0125\u0126\7\60\2\2\u0126")
+        buf.write("\\\3\2\2\2\u0127\u0128\7\60\2\2\u0128\u0129\7\60\2\2\u0129")
+        buf.write("^\3\2\2\2\u012a\u012b\7B\2\2\u012b`\3\2\2\2\u012c\u012d")
+        buf.write("\7%\2\2\u012db\3\2\2\2\u012e\u012f\7\u0080\2\2\u012fd")
+        buf.write("\3\2\2\2\31\2gsy\u0082\u0088\u0090\u0098\u00a2\u00a4\u00a6")
+        buf.write("\u00a8\u00af\u00b2\u00c1\u00c6\u00cd\u00cf\u00d7\u00d9")
+        buf.write("\u00e1\u00e3\u00ea\2")
+        return buf.getvalue()
+
+
+class LexBasic(Lexer):
+
+    atn = ATNDeserializer().deserialize(serializedATN())
+
+    decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ]
+
+
+
+    modeNames = [ u"DEFAULT_MODE" ]
+
+    literalNames = [ u"<INVALID>",
+ ]
+
+    symbolicNames = [ u"<INVALID>",
+ ]
+
+    ruleNames = [ "Ws", "Hws", "Vws", "BlockComment", "DocComment", "LineComment", 
+                  "EscSeq", "EscAny", "UnicodeEsc", "DecimalNumeral", "HexDigit", 
+                  "DecDigit", "BoolLiteral", "CharLiteral", "SQuoteLiteral", 
+                  "DQuoteLiteral", "USQuoteLiteral", "NameChar", "NameStartChar", 
+                  "Int", "Esc", "Colon", "DColon", "SQuote", "DQuote", "LParen", 
+                  "RParen", "LBrace", "RBrace", "LBrack", "RBrack", "RArrow", 
+                  "Lt", "Gt", "Equal", "Question", "Star", "Plus", "PlusAssign", 
+                  "Underscore", "Pipe", "Dollar", "Comma", "Semi", "Dot", 
+                  "Range", "At", "Pound", "Tilde" ]
+
+    grammarFileName = "LexBasic.g4"
+
+    def __init__(self, input=None):
+        super().__init__(input)
+        self.checkVersion("4.5")
+        self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache())
+        self._actions = None
+        self._predicates = None
+
+

tools/grammar-analysis/LexerAdaptor.py

@@ -0,0 +1,66 @@
+from antlr4 import *
+
+
+class LexerAdaptor(Lexer):
+
+    """
+      Track whether we are inside of a rule and whether it is lexical parser. _currentRuleType==Token.INVALID_TYPE
+      means that we are outside of a rule. At the first sign of a rule name reference and _currentRuleType==invalid, we
+      can assume that we are starting a parser rule. Similarly, seeing a token reference when not already in rule means
+      starting a token rule. The terminating ';' of a rule, flips this back to invalid type.
+
+      This is not perfect logic but works. For example, "grammar T;" means that we start and stop a lexical rule for
+      the "T;". Dangerous but works.
+
+      The whole point of this state information is to distinguish between [..arg actions..] and [charsets]. Char sets
+      can only occur in lexical rules and arg actions cannot occur.
+    """
+
+    _currentRuleType = Token.INVALID_TYPE
+
+    def __init__(self, inp):
+        Lexer.__init__(self, inp)
+
+    def getCurrentRuleType(self):
+        return self._currentRuleType
+
+    def setCurrentRuleType(self, ruleType):
+        self._currentRuleType = ruleType
+
+    def handleBeginArgument(self):
+        if self.inLexerRule():
+            self.pushMode(self.LexerCharSet)
+            self.more()
+        else:
+            self.pushMode(self.Argument)
+
+    def handleEndArgument(self):
+        self.popMode()
+        if len(self._modeStack) > 0:
+            self.setType(self.ARGUMENT_CONTENT)
+
+    def handleEndAction(self):
+        self.popMode()
+        if len(self._modeStack) > 0:
+            self.setType(self.ACTION_CONTENT)
+
+    def emit(self):
+        if self._type == self.ID:
+            firstChar = self._input.getText(self._tokenStartCharIndex, self._tokenStartCharIndex)
+            if firstChar[0].isupper():
+                self._type = self.TOKEN_REF
+            else:
+                self._type = self.RULE_REF
+
+            if self._currentRuleType == Token.INVALID_TYPE:  # if outside of rule def
+                self._currentRuleType = self._type  # set to inside lexer or parser rule
+
+        elif self._type == self.SEMI:  # exit rule def
+            self._currentRuleType = Token.INVALID_TYPE
+        return Lexer.emit(self)
+
+    def inLexerRule(self):
+        return self._currentRuleType == self.TOKEN_REF
+
+    def inParserRule(self):  # not used, but added for clarity
+        return self._currentRuleType == self.RULE_REF

tools/grammar-analysis/README.md

@@ -0,0 +1,6 @@
+# grammar-analysis
+
+This is meant to be used as a tool assisting the creation of Droplet modes. It parses antlr4 grammar files and searches for
+rules that can have other rules as their only children, for instance in C mode,
+`additiveExpression -> multiplicativeExpression`. These are used for droppability rule computations. In the future it may
+need to support more detailed analysis.