Skip to content

Commit 2967c8f

Browse files
KvanTTTKvanTTT
committed
Use int literals instead of refs for Python and JavaScript
Update getMultiTokenAlternativeDescriptor test fixes #3703 Signed-off-by: Ivan Kochurkin <[email protected]>
1 parent e137adf commit 2967c8f

File tree

10 files changed

+60
-21
lines changed

10 files changed

+60
-21
lines changed

‎runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java

+8-3
Original file line numberDiff line numberDiff line change
@@ -154,10 +154,11 @@ private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() {
154154
private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
155155
final int tokensCount = 64;
156156

157-
StringBuilder rule = new StringBuilder("t: ");
157+
StringBuilder rule = new StringBuilder("r1: ");
158158
StringBuilder tokens = new StringBuilder();
159159
StringBuilder input = new StringBuilder();
160160
StringBuilder output = new StringBuilder();
161+
161162
for (int i = 0; i < tokensCount; i++) {
162163
String currentToken = "T" + i;
163164
rule.append(currentToken);
@@ -170,17 +171,21 @@ private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
170171
input.append(currentToken).append(" ");
171172
output.append(currentToken);
172173
}
174+
String currentToken = "T" + tokensCount;
175+
tokens.append(currentToken).append(": '").append(currentToken).append("';\n");
176+
input.append(currentToken).append(" ");
177+
output.append(currentToken);
173178

174179
String grammar = "grammar P;\n" +
175-
"r: t+ EOF {<writeln(\"$text\")>};\n" +
180+
"r: (r1 | T" + tokensCount + ")+ EOF {<writeln(\"$text\")>};\n" +
176181
rule + "\n" +
177182
tokens + "\n" +
178183
"WS: [ ]+ -> skip;";
179184

180185
return new RuntimeTestDescriptor(
181186
GrammarType.Parser,
182187
"MultiTokenAlternative",
183-
"https://github.com/antlr/antlr4/issues/3698",
188+
"https://github.com/antlr/antlr4/issues/3698, https://github.com/antlr/antlr4/issues/3703",
184189
input.toString(),
185190
output + "\n",
186191
"",

‎tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg

+5-5
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
246246

247247
<currentRule.escapedName>(<currentRule.args:{a | <a.escapedName>}; separator=", ">) {
248248
let localctx = new <currentRule.ctxType>(this, this._ctx, this.state<currentRule.args:{a | , <a.escapedName>}>);
249-
this.enterRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>);
249+
this.enterRule(localctx, <currentRule.startState>, <currentRule.index>);
250250
<namedActions.init>
251251
<locals; separator="\n">
252252
try {
@@ -285,7 +285,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,
285285
let localctx = new <currentRule.ctxType>(this, this._ctx, _parentState<args:{a | , <a.escapedName>}>);
286286
let _prevctx = localctx;
287287
const _startState = <currentRule.startState>;
288-
this.enterRecursionRule(localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>, _p);
288+
this.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p);
289289
<namedActions.init>
290290
<locals; separator="\n">
291291
try {
@@ -477,11 +477,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
477477

478478
// produces more efficient bytecode when bits.ttypes contains at most two items
479479
bitsetInlineComparison(s, bits) ::= <%
480-
<bits.ttypes:{ttype | <s.varName>===<parser.name>.<ttype>}; separator=" || ">
480+
<bits.ttypes:{ttype | <s.varName>===<ttype>}; separator=" || ">
481481
%>
482482

483483
cases(ttypes) ::= <<
484-
<ttypes:{t | case <parser.name>.<t>:}; separator="\n">
484+
<ttypes:{t | case <t>:}; separator="\n">
485485
>>
486486

487487
InvokeRule(r, argExprsChunks) ::= <<
@@ -491,7 +491,7 @@ this.state = <r.stateNumber>;
491491

492492
MatchToken(m) ::= <<
493493
this.state = <m.stateNumber>;
494-
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>this.match(<parser.name>.<m.name>);
494+
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>this.match(<m.ttype>);
495495
>>
496496

497497
MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

‎tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg

+5-5
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
253253
def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName>}>):
254254

255255
localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, self.state<currentRule.args:{a | , <a.escapedName>}>)
256-
self.enterRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>)
256+
self.enterRule(localctx, <currentRule.startState>, <currentRule.index>)
257257
<namedActions.init>
258258
<locals; separator="\n">
259259
try:
@@ -288,7 +288,7 @@ def <currentRule.escapedName>(self, _p=0<if(currentRule.args)>, <args:{a | , <a>
288288
localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, _parentState<args:{a | , <a.escapedName>}>)
289289
_prevctx = localctx
290290
_startState = <currentRule.startState>
291-
self.enterRecursionRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>, _p)
291+
self.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p)
292292
<namedActions.init>
293293
<locals; separator="\n">
294294
try:
@@ -471,11 +471,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
471471

472472
// produces more efficient bytecode when bits.ttypes contains at most two items
473473
bitsetInlineComparison(s, bits) ::= <%
474-
<bits.ttypes:{ttype | <s.varName>==<parser.name>.<ttype>}; separator=" or ">
474+
<bits.ttypes:{ttype | <s.varName>==<ttype>}; separator=" or ">
475475
%>
476476

477477
cases(ttypes) ::= <<
478-
if token in [<ttypes:{t | <parser.name>.<t>}; separator=", ">]:
478+
if token in [<ttypes:{t | <t>}; separator=", ">]:
479479
>>
480480

481481
InvokeRule(r, argExprsChunks) ::= <<
@@ -485,7 +485,7 @@ self.state = <r.stateNumber>
485485

486486
MatchToken(m) ::= <<
487487
self.state = <m.stateNumber>
488-
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<parser.name>.<m.name>)
488+
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<m.ttype>)
489489
>>
490490

491491
MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

‎tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg

+5-5
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina
266266
def <currentRule.escapedName>(self<currentRule.args:{a | , <a.escapedName><if(a.type)>:<a.type><endif>}>):
267267

268268
localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, self.state<currentRule.args:{a | , <a.escapedName>}>)
269-
self.enterRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>)
269+
self.enterRule(localctx, <currentRule.startState>, <currentRule.index>)
270270
<namedActions.init>
271271
<locals; separator="\n">
272272
try:
@@ -301,7 +301,7 @@ def <currentRule.escapedName>(self, _p:int=0<if(currentRule.args)>, <args:{a | ,
301301
localctx = <parser.name>.<currentRule.ctxType>(self, self._ctx, _parentState<args:{a | , <a.escapedName>}>)
302302
_prevctx = localctx
303303
_startState = <currentRule.startState>
304-
self.enterRecursionRule(localctx, <currentRule.startState>, self.RULE_<currentRule.name>, _p)
304+
self.enterRecursionRule(localctx, <currentRule.startState>, <currentRule.index>, _p)
305305
<namedActions.init>
306306
<locals; separator="\n">
307307
try:
@@ -484,11 +484,11 @@ offsetShiftType(shiftAmount, offset) ::= <%
484484

485485
// produces more efficient bytecode when bits.ttypes contains at most two items
486486
bitsetInlineComparison(s, bits) ::= <%
487-
<bits.ttypes:{ttype | <s.varName>==<parser.name>.<ttype>}; separator=" or ">
487+
<bits.ttypes:{ttype | <s.varName>==<ttype>}; separator=" or ">
488488
%>
489489

490490
cases(ttypes) ::= <<
491-
if token in [<ttypes:{t | <parser.name>.<t>}; separator=", ">]:
491+
if token in [<ttypes:{t | <t>}; separator=", ">]:
492492
>>
493493

494494
InvokeRule(r, argExprsChunks) ::= <<
@@ -498,7 +498,7 @@ self.state = <r.stateNumber>
498498

499499
MatchToken(m) ::= <<
500500
self.state = <m.stateNumber>
501-
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<parser.name>.<m.name>)
501+
<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>self.match(<m.ttype>)
502502
>>
503503

504504
MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"

‎tool/src/org/antlr/v4/codegen/Target.java

+2
Original file line numberDiff line numberDiff line change
@@ -655,4 +655,6 @@ public boolean isATNSerializedAsInts() {
655655

656656
/** @since 4.6 */
657657
public boolean needsHeader() { return false; } // Override in targets that need header files.
658+
659+
public boolean supportsConstants() { return true; }
658660
}

‎tool/src/org/antlr/v4/codegen/model/Choice.java

+17-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,13 @@
77
package org.antlr.v4.codegen.model;
88

99
import org.antlr.v4.codegen.OutputModelFactory;
10+
import org.antlr.v4.codegen.Target;
1011
import org.antlr.v4.codegen.model.decl.Decl;
1112
import org.antlr.v4.codegen.model.decl.TokenTypeDecl;
1213
import org.antlr.v4.misc.Utils;
14+
import org.antlr.v4.runtime.misc.IntegerList;
1315
import org.antlr.v4.runtime.misc.IntervalSet;
16+
import org.antlr.v4.tool.Grammar;
1417
import org.antlr.v4.tool.ast.GrammarAST;
1518

1619
import java.util.ArrayList;
@@ -46,9 +49,21 @@ public void addPreambleOp(SrcOp op) {
4649
}
4750

4851
public List<String[]> getAltLookaheadAsStringLists(IntervalSet[] altLookSets) {
49-
List<String[]> altLook = new ArrayList<String[]>();
52+
List<String[]> altLook = new ArrayList<>();
53+
Target target = factory.getGenerator().getTarget();
54+
Grammar grammar = factory.getGrammar();
5055
for (IntervalSet s : altLookSets) {
51-
altLook.add(factory.getGenerator().getTarget().getTokenTypesAsTargetLabels(factory.getGrammar(), s.toArray()));
56+
if (target.supportsConstants()) {
57+
altLook.add(target.getTokenTypesAsTargetLabels(grammar, s.toArray()));
58+
}
59+
else {
60+
IntegerList list = s.toIntegerList();
61+
String[] labels = new String[list.size()];
62+
for (int i = 0; i < labels.length; i++) {
63+
labels[i] = String.valueOf(list.get(i));
64+
}
65+
altLook.add(labels);
66+
}
5267
}
5368
return altLook;
5469
}

‎tool/src/org/antlr/v4/codegen/model/TestSetInline.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,14 @@ private static Bitset[] createBitsets(OutputModelFactory factory,
4949
bitsetList.add(current);
5050
}
5151

52-
current.addToken(ttype, target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
52+
String name;
53+
if (target.supportsConstants()) {
54+
name = target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype);
55+
}
56+
else {
57+
name = String.valueOf(ttype);
58+
}
59+
current.addToken(ttype, name);
5360
}
5461

5562
return bitsetList.toArray(new Bitset[0]);

‎tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java

+4
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,8 @@ public boolean supportsOverloadedMethods() {
7171
public boolean isATNSerializedAsInts() {
7272
return true;
7373
}
74+
75+
76+
@Override
77+
public boolean supportsConstants() { return false; }
7478
}

‎tool/src/org/antlr/v4/codegen/target/Python2Target.java

+3
Original file line numberDiff line numberDiff line change
@@ -88,5 +88,8 @@ public boolean wantsBaseVisitor() {
8888
public boolean supportsOverloadedMethods() {
8989
return false;
9090
}
91+
92+
@Override
93+
public boolean supportsConstants() { return false; }
9194
}
9295

‎tool/src/org/antlr/v4/codegen/target/Python3Target.java

+3
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,7 @@ public boolean wantsBaseVisitor() {
8787
public boolean supportsOverloadedMethods() {
8888
return false;
8989
}
90+
91+
@Override
92+
public boolean supportsConstants() { return false; }
9093
}

0 commit comments

Comments
 (0)