ScriptTokenizer thinks print is IdentifierToken

[xllifi]

This code

IEnumerable<Token> tkns = [];
tkns = tkns.Concat(ScriptTokenizer.Tokenize("""print("Hello world!")""", 1));
foreach (var tkn in tkns) yield return tkn;

outputs this

1111: Token(Newline, 1)
1112: IdentifierToken(Identifier, 114, print)
1113: Token(ParenthesisOpen, )
1114: ConstantToken(Constant, 341, StringVariant(Hello world!))
1115: Token(ParenthesisClose, )
1116: Token(Newline, 1)

See token 1112? That's wrong.

Considered solution

Use a dictionary for GDScript built-in function > BuiltinFunc. You can't use functions as identifiers in GDScript anyway.

A dictionary I came up with (click to reveal/hide)

Last one is empty and commented out because I couldn't figure out what it corresponds to.

{
  ["sin"] = "MathSin",
  ["cos"] = "MathCos",
  ["tan"] = "MathTan",
  ["sinh"] = "MathSinh",
  ["cosh"] = "MathCosh",
  ["tanh"] = "MathTanh",
  ["asin"] = "MathAsin",
  ["acos"] = "MathAcos",
  ["atan"] = "MathAtan",
  ["atan2"] = "MathAtan2",
  ["sqrt"] = "MathSqrt",
  ["fmod"] = "MathFmod",
  ["fposmod"] = "MathFposmod",
  ["posmod"] = "MathPosmod",
  ["floor"] = "MathFloor",
  ["ceil"] = "MathCeil",
  ["round"] = "MathRound",
  ["abs"] = "MathAbs",
  ["sign"] = "MathSign",
  ["pow"] = "MathPow",
  ["log"] = "MathLog",
  ["exp"] = "MathExp",
  ["is_nan"] = "MathIsnan",
  ["is_inf"] = "MathIsinf",
  ["is_equal_approx"] = "MathIsequalapprox",
  ["is_zero_approx"] = "MathIszeroapprox",
  ["ease"] = "MathEase",
  ["decimals"] = "MathDecimals",
  ["step_decimals"] = "MathStepDecimals",
  ["stepify"] = "MathStepify",
  ["lerp"] = "MathLerp",
  ["lerp_angle"] = "MathLerpAngle",
  ["inverse_lerp"] = "MathInverseLerp",
  ["range_lerp"] = "MathRangeLerp",
  ["smoothstep"] = "MathSmoothstep",
  ["move_toward"] = "MathMoveToward",
  ["dectime"] = "MathDectime",
  ["randomize"] = "MathRandomize",
  ["randi"] = "MathRand",
  ["randf"] = "MathRandf",
  ["rand_range"] = "MathRandom",
  ["seed"] = "MathSeed",
  ["rand_seed"] = "MathRandseed",
  ["deg2rad"] = "MathDeg2Rad",
  ["rad2deg"] = "MathRad2Deg",
  ["linear2db"] = "MathLinear2Db",
  ["db2linear"] = "MathDb2Linear",
  ["polar2cartesian"] = "MathPolar2Cartesian",
  ["cartesian2polar"] = "MathCartesian2Polar",
  ["wrapi"] = "MathWrap",
  ["wrapf"] = "MathWrapf",
  ["max"] = "LogicMax",
  ["min"] = "LogicMin",
  ["clamp"] = "LogicClamp",
  ["nearest_po2"] = "LogicNearestPo2",
  ["weakref"] = "ObjWeakref",
  ["funcref"] = "FuncFuncref",
  ["convert"] = "TypeConvert",
  ["typeof"] = "TypeOf",
  ["type_exists"] = "TypeExists",
  ["char"] = "TextChar",
  ["ord"] = "TextOrd",
  ["str"] = "TextStr",
  ["print"] = "TextPrint",
  ["printt"] = "TextPrintTabbed",
  ["prints"] = "TextPrintSpaced",
  ["printerr"] = "TextPrinterr",
  ["printraw"] = "TextPrintraw",
  ["print_debug"] = "TextPrintDebug",
  ["push_error"] = "PushError",
  ["push_warning"] = "PushWarning",
  ["var2str"] = "VarToStr",
  ["str2var"] = "StrToVar",
  ["var2bytes"] = "VarToBytes",
  ["bytes2var"] = "BytesToVar",
  ["range"] = "GenRange",
  ["load"] = "ResourceLoad",
  ["inst2dict"] = "Inst2Dict",
  ["dict2inst"] = "Dict2Inst",
  ["validate_json"] = "ValidateJson",
  ["parse_json"] = "ParseJson",
  ["to_json"] = "ToJson",
  ["hash"] = "Hash",
  ["Color8"] = "Color8",
  ["ColorN"] = "Colorn",
  ["print_stack"] = "PrintStack",
  ["get_stack"] = "GetStack",
  ["instance_from_id"] = "InstanceFromId",
  ["len"] = "Len",
  ["is_instance_valid"] = "IsInstanceValid",
  ["deep_equal"] = "DeepEqual"
  // [""] = "FuncMax"
}

Temporary solution

(for GDWeave dev) Link this issue in MODS.md.
(for mod dev) Use that dictionary yourself. E.g. replace print to TextPrint in your code:

IEnumerable<Token> tkns = [];
tkns = tkns.Concat(ScriptTokenizer.Tokenize("""TextPrint("Hello world!")""", 1));
foreach (var tkn in tkns) yield return tkn;