Skip to content

Commit

Permalink
Fix partial parsing errors with heredoc bodies.
Browse files Browse the repository at this point in the history
  • Loading branch information
@drgrice1
drgrice1 committed Oct 9, 2024
1 parent b0065a8 commit ab392b8
Show file tree
Hide file tree
Showing 13 changed files with 568 additions and 446 deletions.
2 changes: 1 addition & 1 deletion src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export const pgLanguage = LRLanguage.define({
IfStatement: continuedIndent({ except: /^\s*({|else\b|elsif\b)/ }),
Block: delimitedIndent({ closing: '}' }),
'StringSingleQuoted StringQQuoted StringDoubleQuoted StringQqQuoted': () => null,
'InterpolatedHeredocBody UninterpolatedHeredocBody': () => null,
'StatementEnd InterpolatedHeredocBody UninterpolatedHeredocBody': () => null,
Statement: continuedIndent(),
'PGMLBlock PGTextBlock LaTeXImageCode': flatIndent
}),
Expand Down
13 changes: 7 additions & 6 deletions src/pg.grammar
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,7 @@ statement[@isGroup=Statement] {
(scope)?
kw<"sub">
FunctionName { !bareword Identifier }
(
FunctionPrototype FunctionAttribute* |
FunctionAttribute* FunctionPrototypeAttribute FunctionAttribute*
)?
(FunctionPrototype FunctionAttribute* | FunctionAttribute* FunctionPrototypeAttribute FunctionAttribute*)?
(semicolon | Block)
} |
SingleLineStatement { (expression | ReturnExpression) statementModifier semicolon } |
Expand Down Expand Up @@ -338,7 +335,11 @@ builtinArrayFunction { (kw<"grep"> | kw<"map"> | kw<"join"> | kw<"sort"> | kw<"u

scope { kw<"our"> | kw<"state"> | kw<"my"> | kw<"local"> }

semicolon { ";" | automaticSemicolon }
// This makes it so that a heredoc body is placed inside the ExpressionStatement containing the HeredocInitializer that
// starts it. This is important because when a partial parse occurs this ensures the partial parse goes back far enough
// to see the HeredocInitializer and correctly parses the heredoc body.
@skip {} { StatementEnd { ";" (InterpolatedHeredocBody | UninterpolatedHeredocBody)* statementEnd } }
semicolon { StatementEnd | automaticSemicolon }

@skip {} {
StringDoubleQuoted { '"' interpolationContent* '"' }
Expand Down Expand Up @@ -460,7 +461,7 @@ commaSep1<content> { !atLeastOne content (commaOperator content?)* }
maybeParens<content> { !noParens content | "(" !parens content ")" }

@context contextTracker from "./tokens.js"
@external tokens semicolon from "./tokens" { automaticSemicolon }
@external tokens semicolon from "./tokens" { automaticSemicolon, statementEnd }
@external tokens unrestrictedIdentifier from "./tokens" { UnrestrictedIdentifier }
@external tokens specialScalarVariable from "./tokens" { SpecialScalarVariable }
@external tokens builtinOperator from "./tokens" { NamedUnaryOperator, ListOperator, PGOperator }
Expand Down
1 change: 1 addition & 0 deletions src/pg.grammar.terms.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export declare const automaticSemicolon: number,
statementEnd: number,
UnrestrictedIdentifier: number,
SpecialScalarVariable: number,
NamedUnaryOperator: number,
Expand Down
14 changes: 13 additions & 1 deletion src/tokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { namedUnaryOperators, listOperators } from './perl-operators';
import { pgOperators } from './pg-variables';
import {
automaticSemicolon,
statementEnd,
UnrestrictedIdentifier,
SpecialScalarVariable,
NamedUnaryOperator,
Expand Down Expand Up @@ -337,7 +338,18 @@ export const contextTracker = new ContextTracker<Context>({
});

export const semicolon = new ExternalTokenizer((input, stack) => {
if (stack.canShift(automaticSemicolon) && input.next != 59 /* ; */ && (input.next < 0 || input.next == 125) /* } */)
if (
stack.canShift(statementEnd) &&
!heredocQueue.length &&
(input.peek(-1) == 59 /* ; */ ||
(input.peek(-1) != 59 /* ; */ && (input.next < 0 || input.next == 10))) /* \n */
) {
input.acceptToken(statementEnd, 0);
} else if (
stack.canShift(automaticSemicolon) &&
input.next != 59 /* ; */ &&
(input.next < 0 || input.next == 125) /* } */
)
input.acceptToken(automaticSemicolon);
});

Expand Down
4 changes: 2 additions & 2 deletions test/latex-image.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ END_TIKZ
Program(
ExpressionStatement(
Assignment(ScalarVariable, "=", CallExpression(FunctionName(Identifier), ParenthesizedArguments("(", ")"))),
";"
StatementEnd(";")
),
LaTeXImageCode(
ScalarVariable,
Expand All @@ -38,7 +38,7 @@ END_LATEX_IMAGE
Program(
ExpressionStatement(
Assignment(ScalarVariable, "=", CallExpression(FunctionName(Identifier), ParenthesizedArguments("(", ")"))),
";"
StatementEnd(";")
),
LaTeXImageCode(
ScalarVariable,
Expand Down
142 changes: 83 additions & 59 deletions test/perl-basic-operations.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,34 @@ $var--;
==>

Program(
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, ArithOp, Integer), ArithOp, Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, ArithOp, Integer), ArithOp, Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "*", Integer), "*", Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "/", Integer), "/", Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "%", Integer), "%", Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, ArithOp, Integer)), ";"),
ExpressionStatement(BinaryExpression(List("(", Integer, ")"), x, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, "*", Integer)), ";"),
ExpressionStatement(BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, "/", Integer)), ";"),
ExpressionStatement(UnaryExpression(ArithOp, Integer), ";"),
ExpressionStatement(UnaryExpression(ArithOp, Integer), ";"),
ExpressionStatement(UnaryExpression(LogicOp, Integer), ";"),
ExpressionStatement(UpdateExpression(ArithOp, ScalarVariable), ";"),
ExpressionStatement(UpdateExpression(ArithOp, ScalarVariable), ";"),
ExpressionStatement(UpdateExpression(ScalarVariable, ArithOp), ";"),
ExpressionStatement(UpdateExpression(ScalarVariable, ArithOp), ";"),
ExpressionStatement(
BinaryExpression(BinaryExpression(Integer, ArithOp, Integer), ArithOp, Integer),
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(Integer, ArithOp, Integer), ArithOp, Integer),
StatementEnd(";")
),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "*", Integer), "*", Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "/", Integer), "/", Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, "%", Integer), "%", Integer), StatementEnd(";")),
ExpressionStatement(
BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, ArithOp, Integer)),
StatementEnd(";")
),
ExpressionStatement(BinaryExpression(List("(", Integer, ")"), x, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, "*", Integer)), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, ArithOp, BinaryExpression(Integer, "/", Integer)), StatementEnd(";")),
ExpressionStatement(UnaryExpression(ArithOp, Integer), StatementEnd(";")),
ExpressionStatement(UnaryExpression(ArithOp, Integer), StatementEnd(";")),
ExpressionStatement(UnaryExpression(LogicOp, Integer), StatementEnd(";")),
ExpressionStatement(UpdateExpression(ArithOp, ScalarVariable), StatementEnd(";")),
ExpressionStatement(UpdateExpression(ArithOp, ScalarVariable), StatementEnd(";")),
ExpressionStatement(UpdateExpression(ScalarVariable, ArithOp), StatementEnd(";")),
ExpressionStatement(UpdateExpression(ScalarVariable, ArithOp), StatementEnd(";")),
ExpressionStatement(
BinaryExpression(List("(", BinaryExpression(Float, ArithOp, Float), ")"), "*", Integer),
";"
StatementEnd(";")
)
)

Expand All @@ -62,19 +71,19 @@ Program(
==>

Program(
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, lt, StringSingleQuoted), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, gt, StringSingleQuoted), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, le, StringSingleQuoted), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, eq, StringSingleQuoted), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, ne, StringSingleQuoted), ";"),
ExpressionStatement(BinaryExpression(StringSingleQuoted, cmp, StringSingleQuoted), ";")
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, lt, StringSingleQuoted), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, gt, StringSingleQuoted), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, le, StringSingleQuoted), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, CompareOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, eq, StringSingleQuoted), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, ne, StringSingleQuoted), StatementEnd(";")),
ExpressionStatement(BinaryExpression(StringSingleQuoted, cmp, StringSingleQuoted), StatementEnd(";"))
)

# logic operations
Expand All @@ -101,57 +110,60 @@ $a or $b or $c;
==>

Program(
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), StatementEnd(";")),
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), StatementEnd(";")),
ExpressionStatement(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), StatementEnd(";")),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), LogicOp, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(ScalarVariable, LogicOp, BinaryExpression(ScalarVariable, LogicOp, ScalarVariable)),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(ScalarVariable, LogicOp, BinaryExpression(ScalarVariable, LogicOp, ScalarVariable)),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), LogicOp, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), LogicOp, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, LogicOp, ScalarVariable), LogicOp, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(BinaryExpression(ScalarVariable, and, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, or, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, xor, ScalarVariable), ";"),
ExpressionStatement(BinaryExpression(ScalarVariable, and, ScalarVariable), StatementEnd(";")),
ExpressionStatement(BinaryExpression(ScalarVariable, or, ScalarVariable), StatementEnd(";")),
ExpressionStatement(BinaryExpression(ScalarVariable, xor, ScalarVariable), StatementEnd(";")),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, and, ScalarVariable), or, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(ScalarVariable, or, BinaryExpression(ScalarVariable, and, ScalarVariable)),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, and, ScalarVariable), xor, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(ScalarVariable, xor, BinaryExpression(ScalarVariable, and, ScalarVariable)),
";"
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, and, ScalarVariable), and, ScalarVariable),
";"
StatementEnd(";")
),
ExpressionStatement(BinaryExpression(BinaryExpression(ScalarVariable, or, ScalarVariable), or, ScalarVariable), ";")
ExpressionStatement(
BinaryExpression(BinaryExpression(ScalarVariable, or, ScalarVariable), or, ScalarVariable),
StatementEnd(";")
)
)

# bitwise operations
Expand All @@ -171,15 +183,27 @@ Program(
==>

Program(
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, BitOp("&"), Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, "<<", Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp("&"), Integer), BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, BitOp, BinaryExpression(Integer, BitOp("&"), Integer)), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp("&"), Integer), BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(Integer, BitOp, BinaryExpression(Integer, BitOp("&"), Integer)), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp, Integer), BitOp, Integer), ";"),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp, Integer), BitOp, Integer), ";")
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, BitOp("&"), Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, BitOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(Integer, "<<", Integer), StatementEnd(";")),
ExpressionStatement(
BinaryExpression(BinaryExpression(Integer, BitOp("&"), Integer), BitOp, Integer),
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(Integer, BitOp, BinaryExpression(Integer, BitOp("&"), Integer)),
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(BinaryExpression(Integer, BitOp("&"), Integer), BitOp, Integer),
StatementEnd(";")
),
ExpressionStatement(
BinaryExpression(Integer, BitOp, BinaryExpression(Integer, BitOp("&"), Integer)),
StatementEnd(";")
),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp, Integer), BitOp, Integer), StatementEnd(";")),
ExpressionStatement(BinaryExpression(BinaryExpression(Integer, BitOp, Integer), BitOp, Integer), StatementEnd(";"))
)
Loading

0 comments on commit ab392b8

Please sign in to comment.