Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/xlwt/antlr.py: 25%

1from __future__ import print_function

2## This file is part of PyANTLR. See LICENSE.txt for license

3## details..........Copyright (C) Wolfgang Haefelinger, 2004.

5## This file was copied for use with xlwt from the 2.7.7 ANTLR distribution. Yes, it

6## says 2.7.5 below. The 2.7.5 distribution version didn't have a

7## version in it.

9## Here is the contents of the ANTLR 2.7.7 LICENSE.txt referred to above.

11# SOFTWARE RIGHTS

12#

13# ANTLR 1989-2006 Developed by Terence Parr

14# Partially supported by University of San Francisco & jGuru.com

15#

16# We reserve no legal rights to the ANTLR--it is fully in the

17# public domain. An individual or company may do whatever

18# they wish with source code distributed with ANTLR or the

19# code generated by ANTLR, including the incorporation of

20# ANTLR, or its output, into commerical software.

21#

22# We encourage users to develop software with ANTLR. However,

23# we do ask that credit is given to us for developing

24# ANTLR. By "credit", we mean that if you use ANTLR or

25# incorporate any source code into one of your programs

26# (commercial product, research project, or otherwise) that

27# you acknowledge this fact somewhere in the documentation,

28# research report, etc... If you like ANTLR and have

29# developed a nice tool with the output, please mention that

30# you developed it using ANTLR. In addition, we ask that the

31# headers remain intact in our source code. As long as these

32# guidelines are kept, we expect to continue enhancing this

33# system and expect to make other tools available as they are

34# completed.

35#

36# The primary ANTLR guy:

37#

38# Terence Parr

39# parrt@cs.usfca.edu

40# parrt@antlr.org

42## End of contents of the ANTLR 2.7.7 LICENSE.txt ########################

44## get sys module

45import sys

47from .compat import long, basestring, int_types, xrange

49###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

50### global symbols ###

51###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

53### ANTLR Standard Tokens

54SKIP = -1

55INVALID_TYPE = 0

56EOF_TYPE = 1

57EOF = 1

58NULL_TREE_LOOKAHEAD = 3

59MIN_USER_TYPE = 4

61### ANTLR's EOF Symbol

62EOF_CHAR = ''

64###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

65### general functions ###

66###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

68## Version should be automatically derived from configure.in. For now,

69## we need to bump it ourselfs. Don't remove the <version> tags.

70## <version>

71def version():

72 r = {

73 'major' : '2',

74 'minor' : '7',

75 'micro' : '5',

76 'patch' : '' ,

77 'version': '2.7.5'

78 }

79 return r

80## </version>

82def error(fmt,*args):

83 if fmt:

84 print("error: ", fmt % tuple(args))

86def ifelse(cond,_then,_else):

87 if cond :

88 r = _then

89 else:

90 r = _else

91 return r

93def is_string_type(x):

94 # return (isinstance(x,str) or isinstance(x,unicode))

95 # Simplify; xlwt doesn't support Python < 2.3

96 return isinstance(basestring)

98def assert_string_type(x):

99 assert is_string_type(x)

100 pass

101

102###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

103### ANTLR Exceptions ###

104###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

105

106class ANTLRException(Exception):

107

108 def __init__(self, *args):

109 Exception.__init__(self, *args)

110

111

112class RecognitionException(ANTLRException):

113

114 def __init__(self, *args):

115 ANTLRException.__init__(self, *args)

116 self.fileName = None

117 self.line = -1

118 self.column = -1

119 if len(args) >= 2:

120 self.fileName = args[1]

121 if len(args) >= 3:

122 self.line = args[2]

123 if len(args) >= 4:

124 self.column = args[3]

125

126 def __str__(self):

127 buf = ['']

128 if self.fileName:

129 buf.append(self.fileName + ":")

130 if self.line != -1:

131 if not self.fileName:

132 buf.append("line ")

133 buf.append(str(self.line))

134 if self.column != -1:

135 buf.append(":" + str(self.column))

136 buf.append(":")

137 buf.append(" ")

138 return str('').join(buf)

139

140 __repr__ = __str__

141

142

143class NoViableAltException(RecognitionException):

144

145 def __init__(self, *args):

146 RecognitionException.__init__(self, *args)

147 self.token = None

148 self.node = None

149 if isinstance(args[0],AST):

150 self.node = args[0]

151 elif isinstance(args[0],Token):

152 self.token = args[0]

153 else:

154 raise TypeError("NoViableAltException requires Token or AST argument")

155

156 def __str__(self):

157 if self.token:

158 line = self.token.getLine()

159 col = self.token.getColumn()

160 text = self.token.getText()

161 return "unexpected symbol at line %s (column %s): \"%s\"" % (line,col,text)

162 if self.node == ASTNULL:

163 return "unexpected end of subtree"

164 assert self.node

165 ### hackish, we assume that an AST contains method getText

166 return "unexpected node: %s" % (self.node.getText())

167

168 __repr__ = __str__

169

170

171class NoViableAltForCharException(RecognitionException):

172

173 def __init__(self, *args):

174 self.foundChar = None

175 if len(args) == 2:

176 self.foundChar = args[0]

177 scanner = args[1]

178 RecognitionException.__init__(self, "NoViableAlt",

179 scanner.getFilename(),

180 scanner.getLine(),

181 scanner.getColumn())

182 elif len(args) == 4:

183 self.foundChar = args[0]

184 fileName = args[1]

185 line = args[2]

186 column = args[3]

187 RecognitionException.__init__(self, "NoViableAlt",

188 fileName, line, column)

189 else:

190 RecognitionException.__init__(self, "NoViableAlt",

191 '', -1, -1)

192

193 def __str__(self):

194 mesg = "unexpected char: "

195 if self.foundChar >= ' ' and self.foundChar <= '~':

196 mesg += "'" + self.foundChar + "'"

197 elif self.foundChar:

198 mesg += "0x" + hex(ord(self.foundChar)).upper()[2:]

199 else:

200 mesg += "<None>"

201 return mesg

202

203 __repr__ = __str__

204

205

206class SemanticException(RecognitionException):

207

208 def __init__(self, *args):

209 RecognitionException.__init__(self, *args)

210

211

212class MismatchedCharException(RecognitionException):

213

214 NONE = 0

215 CHAR = 1

216 NOT_CHAR = 2

217 RANGE = 3

218 NOT_RANGE = 4

219 SET = 5

220 NOT_SET = 6

221

222 def __init__(self, *args):

223 self.args = args

224 if len(args) == 5:

225 # Expected range / not range

226 if args[3]:

227 self.mismatchType = MismatchedCharException.NOT_RANGE

228 else:

229 self.mismatchType = MismatchedCharException.RANGE

230 self.foundChar = args[0]

231 self.expecting = args[1]

232 self.upper = args[2]

233 self.scanner = args[4]

234 RecognitionException.__init__(self, "Mismatched char range",

235 self.scanner.getFilename(),

236 self.scanner.getLine(),

237 self.scanner.getColumn())

238 elif len(args) == 4 and is_string_type(args[1]):

239 # Expected char / not char

240 if args[2]:

241 self.mismatchType = MismatchedCharException.NOT_CHAR

242 else:

243 self.mismatchType = MismatchedCharException.CHAR

244 self.foundChar = args[0]

245 self.expecting = args[1]

246 self.scanner = args[3]

247 RecognitionException.__init__(self, "Mismatched char",

248 self.scanner.getFilename(),

249 self.scanner.getLine(),

250 self.scanner.getColumn())

251 elif len(args) == 4 and isinstance(args[1], BitSet):

252 # Expected BitSet / not BitSet

253 if args[2]:

254 self.mismatchType = MismatchedCharException.NOT_SET

255 else:

256 self.mismatchType = MismatchedCharException.SET

257 self.foundChar = args[0]

258 self.set = args[1]

259 self.scanner = args[3]

260 RecognitionException.__init__(self, "Mismatched char set",

261 self.scanner.getFilename(),

262 self.scanner.getLine(),

263 self.scanner.getColumn())

264 else:

265 self.mismatchType = MismatchedCharException.NONE

266 RecognitionException.__init__(self, "Mismatched char")

267

268 ## Append a char to the msg buffer. If special,

269 # then show escaped version

270 #

271 def appendCharName(self, sb, c):

272 if not c or c == 65535:

273 # 65535 = (char) -1 = EOF

274 sb.append("'<EOF>'")

275 elif c == '\n':

276 sb.append("'\\n'")

277 elif c == '\r':

278 sb.append("'\\r'");

279 elif c == '\t':

280 sb.append("'\\t'")

281 else:

282 sb.append('\'' + c + '\'')

283

284 ##

285 # Returns an error message with line number/column information

286 #

287 def __str__(self):

288 sb = ['']

289 sb.append(RecognitionException.__str__(self))

290

291 if self.mismatchType == MismatchedCharException.CHAR:

292 sb.append("expecting ")

293 self.appendCharName(sb, self.expecting)

294 sb.append(", found ")

295 self.appendCharName(sb, self.foundChar)

296 elif self.mismatchType == MismatchedCharException.NOT_CHAR:

297 sb.append("expecting anything but '")

298 self.appendCharName(sb, self.expecting)

299 sb.append("'; got it anyway")

300 elif self.mismatchType in [MismatchedCharException.RANGE, MismatchedCharException.NOT_RANGE]:

301 sb.append("expecting char ")

302 if self.mismatchType == MismatchedCharException.NOT_RANGE:

303 sb.append("NOT ")

304 sb.append("in range: ")

305 self.appendCharName(sb, self.expecting)

306 sb.append("..")

307 self.appendCharName(sb, self.upper)

308 sb.append(", found ")

309 self.appendCharName(sb, self.foundChar)

310 elif self.mismatchType in [MismatchedCharException.SET, MismatchedCharException.NOT_SET]:

311 sb.append("expecting ")

312 if self.mismatchType == MismatchedCharException.NOT_SET:

313 sb.append("NOT ")

314 sb.append("one of (")

315 for i in range(len(self.set)):

316 self.appendCharName(sb, self.set[i])

317 sb.append("), found ")

318 self.appendCharName(sb, self.foundChar)

319

320 return str().join(sb).strip()

321

322 __repr__ = __str__

323

324

325class MismatchedTokenException(RecognitionException):

326

327 NONE = 0

328 TOKEN = 1

329 NOT_TOKEN = 2

330 RANGE = 3

331 NOT_RANGE = 4

332 SET = 5

333 NOT_SET = 6

334

335 def __init__(self, *args):

336 self.args = args

337 self.tokenNames = []

338 self.token = None

339 self.tokenText = ''

340 self.node = None

341 if len(args) == 6:

342 # Expected range / not range

343 if args[3]:

344 self.mismatchType = MismatchedTokenException.NOT_RANGE

345 else:

346 self.mismatchType = MismatchedTokenException.RANGE

347 self.tokenNames = args[0]

348 self.expecting = args[2]

349 self.upper = args[3]

350 self.fileName = args[5]

351

352 elif len(args) == 4 and isinstance(args[2], int):

353 # Expected token / not token

354 if args[3]:

355 self.mismatchType = MismatchedTokenException.NOT_TOKEN

356 else:

357 self.mismatchType = MismatchedTokenException.TOKEN

358 self.tokenNames = args[0]

359 self.expecting = args[2]

360

361 elif len(args) == 4 and isinstance(args[2], BitSet):

362 # Expected BitSet / not BitSet

363 if args[3]:

364 self.mismatchType = MismatchedTokenException.NOT_SET

365 else:

366 self.mismatchType = MismatchedTokenException.SET

367 self.tokenNames = args[0]

368 self.set = args[2]

369

370 else:

371 self.mismatchType = MismatchedTokenException.NONE

372 RecognitionException.__init__(self, "Mismatched Token: expecting any AST node", "<AST>", -1, -1)

373

374 if len(args) >= 2:

375 if isinstance(args[1],Token):

376 self.token = args[1]

377 self.tokenText = self.token.getText()

378 RecognitionException.__init__(self, "Mismatched Token",

379 self.fileName,

380 self.token.getLine(),

381 self.token.getColumn())

382 elif isinstance(args[1],AST):

383 self.node = args[1]

384 self.tokenText = str(self.node)

385 RecognitionException.__init__(self, "Mismatched Token",

386 "<AST>",

387 self.node.getLine(),

388 self.node.getColumn())

389 else:

390 self.tokenText = "<empty tree>"

391 RecognitionException.__init__(self, "Mismatched Token",

392 "<AST>", -1, -1)

393

394 def appendTokenName(self, sb, tokenType):

395 if tokenType == INVALID_TYPE:

396 sb.append("<Set of tokens>")

397 elif tokenType < 0 or tokenType >= len(self.tokenNames):

398 sb.append("<" + str(tokenType) + ">")

399 else:

400 sb.append(self.tokenNames[tokenType])

401

402 ##

403 # Returns an error message with line number/column information

404 #

405 def __str__(self):

406 sb = ['']

407 sb.append(RecognitionException.__str__(self))

408

409 if self.mismatchType == MismatchedTokenException.TOKEN:

410 sb.append("expecting ")

411 self.appendTokenName(sb, self.expecting)

412 sb.append(", found " + self.tokenText)

413 elif self.mismatchType == MismatchedTokenException.NOT_TOKEN:

414 sb.append("expecting anything but '")

415 self.appendTokenName(sb, self.expecting)

416 sb.append("'; got it anyway")

417 elif self.mismatchType in [MismatchedTokenException.RANGE, MismatchedTokenException.NOT_RANGE]:

418 sb.append("expecting token ")

419 if self.mismatchType == MismatchedTokenException.NOT_RANGE:

420 sb.append("NOT ")

421 sb.append("in range: ")

422 self.appendTokenName(sb, self.expecting)

423 sb.append("..")

424 self.appendTokenName(sb, self.upper)

425 sb.append(", found " + self.tokenText)

426 elif self.mismatchType in [MismatchedTokenException.SET, MismatchedTokenException.NOT_SET]:

427 sb.append("expecting ")

428 if self.mismatchType == MismatchedTokenException.NOT_SET:

429 sb.append("NOT ")

430 sb.append("one of (")

431 for i in range(len(self.set)):

432 self.appendTokenName(sb, self.set[i])

433 sb.append("), found " + self.tokenText)

434

435 return str().join(sb).strip()

436

437 __repr__ = __str__

438

439

440class TokenStreamException(ANTLRException):

441

442 def __init__(self, *args):

443 ANTLRException.__init__(self, *args)

444

445

446# Wraps an Exception in a TokenStreamException

447class TokenStreamIOException(TokenStreamException):

448

449 def __init__(self, *args):

450 if args and isinstance(args[0], Exception):

451 io = args[0]

452 TokenStreamException.__init__(self, str(io))

453 self.io = io

454 else:

455 TokenStreamException.__init__(self, *args)

456 self.io = self

457

458

459# Wraps a RecognitionException in a TokenStreamException

460class TokenStreamRecognitionException(TokenStreamException):

461

462 def __init__(self, *args):

463 if args and isinstance(args[0], RecognitionException):

464 recog = args[0]

465 TokenStreamException.__init__(self, str(recog))

466 self.recog = recog

467 else:

468 raise TypeError("TokenStreamRecognitionException requires RecognitionException argument")

469

470 def __str__(self):

471 return str(self.recog)

472

473 __repr__ = __str__

474

475

476class TokenStreamRetryException(TokenStreamException):

477

478 def __init__(self, *args):

479 TokenStreamException.__init__(self, *args)

480

481

482class CharStreamException(ANTLRException):

483

484 def __init__(self, *args):

485 ANTLRException.__init__(self, *args)

486

487

488# Wraps an Exception in a CharStreamException

489class CharStreamIOException(CharStreamException):

490

491 def __init__(self, *args):

492 if args and isinstance(args[0], Exception):

493 io = args[0]

494 CharStreamException.__init__(self, str(io))

495 self.io = io

496 else:

497 CharStreamException.__init__(self, *args)

498 self.io = self

499

500

501class TryAgain(Exception):

502 pass

503

504

505###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

506### Token ###

507###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

508

509class Token(object):

510 SKIP = -1

511 INVALID_TYPE = 0

512 EOF_TYPE = 1

513 EOF = 1

514 NULL_TREE_LOOKAHEAD = 3

515 MIN_USER_TYPE = 4

516

517 def __init__(self,**argv):

518 try:

519 self.type = argv['type']

520 except:

521 self.type = INVALID_TYPE

522 try:

523 self.text = argv['text']

524 except:

525 self.text = "<no text>"

526

527 def isEOF(self):

528 return (self.type == EOF_TYPE)

529

530 def getColumn(self):

531 return 0

532

533 def getLine(self):

534 return 0

535

536 def getFilename(self):

537 return None

538

539 def setFilename(self,name):

540 return self

541

542 def getText(self):

543 return "<no text>"

544

545 def setText(self,text):

546 if is_string_type(text):

547 pass

548 else:

549 raise TypeError("Token.setText requires string argument")

550 return self

551

552 def setColumn(self,column):

553 return self

554

555 def setLine(self,line):

556 return self

557

558 def getType(self):

559 return self.type

560

561 def setType(self,type):

562 if isinstance(type,int):

563 self.type = type

564 else:

565 raise TypeError("Token.setType requires integer argument")

566 return self

567

568 def toString(self):

569 ## not optimal

570 type_ = self.type

571 if type_ == 3:

572 tval = 'NULL_TREE_LOOKAHEAD'

573 elif type_ == 1:

574 tval = 'EOF_TYPE'

575 elif type_ == 0:

576 tval = 'INVALID_TYPE'

577 elif type_ == -1:

578 tval = 'SKIP'

579 else:

580 tval = type_

581 return '["%s",<%s>]' % (self.getText(),tval)

582

583 __str__ = toString

584 __repr__ = toString

585

586### static attribute ..

587Token.badToken = Token( type=INVALID_TYPE, text="<no text>")

588

589if __name__ == "__main__": 589 ↛ 590line 589 didn't jump to line 590, because the condition on line 589 was never true

590 print("testing ..")

591 T = Token.badToken

592 print(T)

593

594###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

595### CommonToken ###

596###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

597

598class CommonToken(Token):

599

600 def __init__(self,**argv):

601 Token.__init__(self,**argv)

602 self.line = 0

603 self.col = 0

604 try:

605 self.line = argv['line']

606 except:

607 pass

608 try:

609 self.col = argv['col']

610 except:

611 pass

612

613 def getLine(self):

614 return self.line

615

616 def getText(self):

617 return self.text

618

619 def getColumn(self):

620 return self.col

621

622 def setLine(self,line):

623 self.line = line

624 return self

625

626 def setText(self,text):

627 self.text = text

628 return self

629

630 def setColumn(self,col):

631 self.col = col

632 return self

633

634 def toString(self):

635 ## not optimal

636 type_ = self.type

637 if type_ == 3:

638 tval = 'NULL_TREE_LOOKAHEAD'

639 elif type_ == 1:

640 tval = 'EOF_TYPE'

641 elif type_ == 0:

642 tval = 'INVALID_TYPE'

643 elif type_ == -1:

644 tval = 'SKIP'

645 else:

646 tval = type_

647 d = {

648 'text' : self.text,

649 'type' : tval,

650 'line' : self.line,

651 'colm' : self.col

652 }

653

654 fmt = '["%(text)s",<%(type)s>,line=%(line)s,col=%(colm)s]'

655 return fmt % d

656

657 __str__ = toString

658 __repr__ = toString

659

660

661if __name__ == '__main__' : 661 ↛ 662line 661 didn't jump to line 662, because the condition on line 661 was never true

662 T = CommonToken()

663 print(T)

664 T = CommonToken(col=15,line=1,text="some text", type=5)

665 print(T)

666 T = CommonToken()

667 T.setLine(1).setColumn(15).setText("some text").setType(5)

668 print(T)

669 print(T.getLine())

670 print(T.getColumn())

671 print(T.getText())

672 print(T.getType())

673

674###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

675### CommonHiddenStreamToken ###

676###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

677

678class CommonHiddenStreamToken(CommonToken):

679 def __init__(self,*args):

680 CommonToken.__init__(self,*args)

681 self.hiddenBefore = None

682 self.hiddenAfter = None

683

684 def getHiddenAfter(self):

685 return self.hiddenAfter

686

687 def getHiddenBefore(self):

688 return self.hiddenBefore

689

690 def setHiddenAfter(self,t):

691 self.hiddenAfter = t

692

693 def setHiddenBefore(self, t):

694 self.hiddenBefore = t

695

696###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

697### Queue ###

698###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

699

700## Shall be a circular buffer on tokens ..

701class Queue(object):

702

703 def __init__(self):

704 self.buffer = [] # empty list

705

706 def append(self,item):

707 self.buffer.append(item)

708

709 def elementAt(self,index):

710 return self.buffer[index]

711

712 def reset(self):

713 self.buffer = []

714

715 def removeFirst(self):

716 self.buffer.pop(0)

717

718 def length(self):

719 return len(self.buffer)

720

721 def __str__(self):

722 return str(self.buffer)

723

724###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

725### InputBuffer ###

726###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

727

728class InputBuffer(object):

729 def __init__(self):

730 self.nMarkers = 0

731 self.markerOffset = 0

732 self.numToConsume = 0

733 self.queue = Queue()

734

735 def __str__(self):

736 return "(%s,%s,%s,%s)" % (

737 self.nMarkers,

738 self.markerOffset,

739 self.numToConsume,

740 self.queue)

741

742 def __repr__(self):

743 return str(self)

744

745 def commit(self):

746 self.nMarkers -= 1

747

748 def consume(self) :

749 self.numToConsume += 1

750

751 ## probably better to return a list of items

752 ## because of unicode. Or return a unicode

753 ## string ..

754 def getLAChars(self) :

755 i = self.markerOffset

756 n = self.queue.length()

757 s = ''

758 while i<n:

759 s += self.queue.elementAt(i)

760 return s

761

762 ## probably better to return a list of items

763 ## because of unicode chars

764 def getMarkedChars(self) :

765 s = ''

766 i = 0

767 n = self.markerOffset

768 while i<n:

769 s += self.queue.elementAt(i)

770 return s

771

772 def isMarked(self) :

773 return self.nMarkers != 0

774

775 def fill(self,k):

776 ### abstract method

777 raise NotImplementedError()

778

779 def LA(self,k) :

780 self.fill(k)

781 return self.queue.elementAt(self.markerOffset + k - 1)

782

783 def mark(self) :

784 self.syncConsume()

785 self.nMarkers += 1

786 return self.markerOffset

787

788 def rewind(self,mark) :

789 self.syncConsume()

790 self.markerOffset = mark

791 self.nMarkers -= 1

792

793 def reset(self) :

794 self.nMarkers = 0

795 self.markerOffset = 0

796 self.numToConsume = 0

797 self.queue.reset()

798

799 def syncConsume(self) :

800 while self.numToConsume > 0:

801 if self.nMarkers > 0:

802 # guess mode -- leave leading characters and bump offset.

803 self.markerOffset += 1

804 else:

805 # normal mode -- remove first character

806 self.queue.removeFirst()

807 self.numToConsume -= 1

808

809###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

810### CharBuffer ###

811###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

812

813class CharBuffer(InputBuffer):

814 def __init__(self,reader):

815 ##assert isinstance(reader,file)

816 super(CharBuffer,self).__init__()

817 ## a reader is supposed to be anything that has

818 ## a method 'read(int)'.

819 self.input = reader

820

821 def __str__(self):

822 base = super(CharBuffer,self).__str__()

823 return "CharBuffer{%s,%s" % (base,str(input))

824

825 def fill(self,amount):

826 try:

827 self.syncConsume()

828 while self.queue.length() < (amount + self.markerOffset) :

829 ## retrieve just one char - what happend at end

830 ## of input?

831 c = self.input.read(1)

832 ### python's behaviour is to return the empty string on

833 ### EOF, ie. no exception whatsoever is thrown. An empty

834 ### python string has the nice feature that it is of

835 ### type 'str' and "not ''" would return true. Contrary,

836 ### one can't do this: '' in 'abc'. This should return

837 ### false, but all we get is then a TypeError as an

838 ### empty string is not a character.

839

840 ### Let's assure then that we have either seen a

841 ### character or an empty string (EOF).

842 assert len(c) == 0 or len(c) == 1

843

844 ### And it shall be of type string (ASCII or UNICODE).

845 assert is_string_type(c)

846

847 ### Just append EOF char to buffer. Note that buffer may

848 ### contain then just more than one EOF char ..

849

850 ### use unicode chars instead of ASCII ..

851 self.queue.append(c)

852 except Exception as e:

853 raise CharStreamIOException(e)

854 ##except: # (mk) Cannot happen ...

855 ##error ("unexpected exception caught ..")

856 ##assert 0

857

858###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

859### LexerSharedInputState ###

860###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

861

862class LexerSharedInputState(object):

863 def __init__(self,ibuf):

864 assert isinstance(ibuf,InputBuffer)

865 self.input = ibuf

866 self.column = 1

867 self.line = 1

868 self.tokenStartColumn = 1

869 self.tokenStartLine = 1

870 self.guessing = 0

871 self.filename = None

872

873 def reset(self):

874 self.column = 1

875 self.line = 1

876 self.tokenStartColumn = 1

877 self.tokenStartLine = 1

878 self.guessing = 0

879 self.filename = None

880 self.input.reset()

881

882 def LA(self,k):

883 return self.input.LA(k)

884

885###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

886### TokenStream ###

887###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

888

889class TokenStream(object):

890 def nextToken(self):

891 pass

892

893 def __iter__(self):

894 return TokenStreamIterator(self)

895

896###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

897### TokenStreamIterator ###

898###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

899

900class TokenStreamIterator(object):

901 def __init__(self,inst):

902 if isinstance(inst,TokenStream):

903 self.inst = inst

904 return

905 raise TypeError("TokenStreamIterator requires TokenStream object")

906

907 def next(self):

908 assert self.inst

909 item = self.inst.nextToken()

910 if not item or item.isEOF():

911 raise StopIteration()

912 return item

913

914###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

915### TokenStreamSelector ###

916###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

917

918class TokenStreamSelector(TokenStream):

919

920 def __init__(self):

921 self._input = None

922 self._stmap = {}

923 self._stack = []

924

925 def addInputStream(self,stream,key):

926 self._stmap[key] = stream

927

928 def getCurrentStream(self):

929 return self._input

930

931 def getStream(self,sname):

932 try:

933 stream = self._stmap[sname]

934 except:

935 raise ValueError("TokenStream " + sname + " not found");

936 return stream;

937

938 def nextToken(self):

939 while 1:

940 try:

941 return self._input.nextToken()

942 except TokenStreamRetryException:

943 ### just retry "forever"

944 pass

945

946 def pop(self):

947 stream = self._stack.pop();

948 self.select(stream);

949 return stream;

950

951 def push(self,arg):

952 self._stack.append(self._input);

953 self.select(arg)

954

955 def retry(self):

956 raise TokenStreamRetryException()

957

958 def select(self,arg):

959 if isinstance(arg,TokenStream):

960 self._input = arg

961 return

962 if is_string_type(arg):

963 self._input = self.getStream(arg)

964 return

965 raise TypeError("TokenStreamSelector.select requires " +

966 "TokenStream or string argument")

967

968###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

969### TokenStreamBasicFilter ###

970###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

971

972class TokenStreamBasicFilter(TokenStream):

973

974 def __init__(self,input):

975

976 self.input = input;

977 self.discardMask = BitSet()

978

979 def discard(self,arg):

980 if isinstance(arg,int):

981 self.discardMask.add(arg)

982 return

983 if isinstance(arg,BitSet):

984 self.discardMark = arg

985 return

986 raise TypeError("TokenStreamBasicFilter.discard requires" +

987 "integer or BitSet argument")

988

989 def nextToken(self):

990 tok = self.input.nextToken()

991 while tok and self.discardMask.member(tok.getType()):

992 tok = self.input.nextToken()

993 return tok

994

995###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

996### TokenStreamHiddenTokenFilter ###

997###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

998

999class TokenStreamHiddenTokenFilter(TokenStreamBasicFilter):

1000

1001 def __init__(self,input):

1002 TokenStreamBasicFilter.__init__(self,input)

1003 self.hideMask = BitSet()

1004 self.nextMonitoredToken = None

1005 self.lastHiddenToken = None

1006 self.firstHidden = None

1007

1008 def consume(self):

1009 self.nextMonitoredToken = self.input.nextToken()

1010

1011 def consumeFirst(self):

1012 self.consume()

1013

1014 p = None;

1015 while self.hideMask.member(self.LA(1).getType()) or \

1016 self.discardMask.member(self.LA(1).getType()):

1017 if self.hideMask.member(self.LA(1).getType()):

1018 if not p:

1019 p = self.LA(1)

1020 else:

1021 p.setHiddenAfter(self.LA(1))

1022 self.LA(1).setHiddenBefore(p)

1023 p = self.LA(1)

1024 self.lastHiddenToken = p

1025 if not self.firstHidden:

1026 self.firstHidden = p

1027 self.consume()

1028

1029 def getDiscardMask(self):

1030 return self.discardMask

1031

1032 def getHiddenAfter(self,t):

1033 return t.getHiddenAfter()

1034

1035 def getHiddenBefore(self,t):

1036 return t.getHiddenBefore()

1037

1038 def getHideMask(self):

1039 return self.hideMask

1040

1041 def getInitialHiddenToken(self):

1042 return self.firstHidden

1043

1044 def hide(self,m):

1045 if isinstance(m,int):

1046 self.hideMask.add(m)

1047 return

1048 if isinstance(m.BitMask):

1049 self.hideMask = m

1050 return

1051

1052 def LA(self,i):

1053 return self.nextMonitoredToken

1054

1055 def nextToken(self):

1056 if not self.LA(1):

1057 self.consumeFirst()

1058

1059 monitored = self.LA(1)

1060

1061 monitored.setHiddenBefore(self.lastHiddenToken)

1062 self.lastHiddenToken = None

1063

1064 self.consume()

1065 p = monitored

1066

1067 while self.hideMask.member(self.LA(1).getType()) or \

1068 self.discardMask.member(self.LA(1).getType()):

1069 if self.hideMask.member(self.LA(1).getType()):

1070 p.setHiddenAfter(self.LA(1))

1071 if p != monitored:

1072 self.LA(1).setHiddenBefore(p)

1073 p = self.lastHiddenToken = self.LA(1)

1074 self.consume()

1075 return monitored

1076

1077###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1078### StringBuffer ###

1079###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1080

1081class StringBuffer:

1082 def __init__(self,string=None):

1083 if string:

1084 self.text = list(string)

1085 else:

1086 self.text = []

1087

1088 def setLength(self,sz):

1089 if not sz :

1090 self.text = []

1091 return

1092 assert sz>0

1093 if sz >= self.length():

1094 return

1095 ### just reset to empty buffer

1096 self.text = self.text[0:sz]

1097

1098 def length(self):

1099 return len(self.text)

1100

1101 def append(self,c):

1102 self.text.append(c)

1103

1104 ### return buffer as string. Arg 'a' is used as index

1105 ## into the buffer and 2nd argument shall be the length.

1106 ## If 2nd args is absent, we return chars till end of

1107 ## buffer starting with 'a'.

1108 def getString(self,a=None,length=None):

1109 if not a :

1110 a = 0

1111 assert a>=0

1112 if a>= len(self.text) :

1113 return ""

1114

1115 if not length:

1116 ## no second argument

1117 L = self.text[a:]

1118 else:

1119 assert (a+length) <= len(self.text)

1120 b = a + length

1121 L = self.text[a:b]

1122 s = ""

1123 for x in L : s += x

1124 return s

1125

1126 toString = getString ## alias

1127

1128 def __str__(self):

1129 return str(self.text)

1130

1131###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1132### Reader ###

1133###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1134

1135## When reading Japanese chars, it happens that a stream returns a

1136## 'char' of length 2. This looks like a bug in the appropriate

1137## codecs - but I'm rather unsure about this. Anyway, if this is

1138## the case, I'm going to split this string into a list of chars

1139## and put them on hold, ie. on a buffer. Next time when called

1140## we read from buffer until buffer is empty.

1141## wh: nov, 25th -> problem does not appear in Python 2.4.0.c1.

1142

1143class Reader(object):

1144 def __init__(self,stream):

1145 self.cin = stream

1146 self.buf = []

1147

1148 def read(self,num):

1149 assert num==1

1150

1151 if len(self.buf):

1152 return self.buf.pop()

1153

1154 ## Read a char - this may return a string.

1155 ## Is this a bug in codecs/Python?

1156 c = self.cin.read(1)

1157

1158 if not c or len(c)==1:

1159 return c

1160

1161 L = list(c)

1162 L.reverse()

1163 for x in L:

1164 self.buf.append(x)

1165

1166 ## read one char ..

1167 return self.read(1)

1168

1169###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1170### CharScanner ###

1171###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1172

1173class CharScanner(TokenStream):

1174 ## class members

1175 NO_CHAR = 0

1176 EOF_CHAR = '' ### EOF shall be the empty string.

1177

1178 def __init__(self, *argv, **kwargs):

1179 super(CharScanner, self).__init__()

1180 self.saveConsumedInput = True

1181 self.tokenClass = None

1182 self.caseSensitive = True

1183 self.caseSensitiveLiterals = True

1184 self.literals = None

1185 self.tabsize = 8

1186 self._returnToken = None

1187 self.commitToPath = False

1188 self.traceDepth = 0

1189 self.text = StringBuffer()

1190 self.hashString = hash(self)

1191 self.setTokenObjectClass(CommonToken)

1192 self.setInput(*argv)

1193

1194 def __iter__(self):

1195 return CharScannerIterator(self)

1196

1197 def setInput(self,*argv):

1198 ## case 1:

1199 ## if there's no arg we default to read from

1200 ## standard input

1201 if not argv:

1202 import sys

1203 self.setInput(sys.stdin)

1204 return

1205

1206 ## get 1st argument

1207 arg1 = argv[0]

1208

1209 ## case 2:

1210 ## if arg1 is a string, we assume it's a file name

1211 ## and open a stream using 2nd argument as open

1212 ## mode. If there's no 2nd argument we fall back to

1213 ## mode '+rb'.

1214 if is_string_type(arg1):

1215 f = open(arg1,"rb")

1216 self.setInput(f)

1217 self.setFilename(arg1)

1218 return

1219

1220 ## case 3:

1221 ## if arg1 is a file we wrap it by a char buffer (

1222 ## some additional checks?? No, can't do this in

1223 ## general).

1224 if isinstance(arg1,file):

1225 self.setInput(CharBuffer(arg1))

1226 return

1227

1228 ## case 4:

1229 ## if arg1 is of type SharedLexerInputState we use

1230 ## argument as is.

1231 if isinstance(arg1,LexerSharedInputState):

1232 self.inputState = arg1

1233 return

1234

1235 ## case 5:

1236 ## check whether argument type is of type input

1237 ## buffer. If so create a SharedLexerInputState and

1238 ## go ahead.

1239 if isinstance(arg1,InputBuffer):

1240 self.setInput(LexerSharedInputState(arg1))

1241 return

1242

1243 ## case 6:

1244 ## check whether argument type has a method read(int)

1245 ## If so create CharBuffer ...

1246 try:

1247 if arg1.read:

1248 rd = Reader(arg1)

1249 cb = CharBuffer(rd)

1250 ss = LexerSharedInputState(cb)

1251 self.inputState = ss

1252 return

1253 except:

1254 pass

1255

1256 ## case 7:

1257 ## raise wrong argument exception

1258 raise TypeError(argv)

1259

1260 def setTabSize(self,size) :

1261 self.tabsize = size

1262

1263 def getTabSize(self) :

1264 return self.tabsize

1265

1266 def setCaseSensitive(self,t) :

1267 self.caseSensitive = t

1268

1269 def setCommitToPath(self,commit) :

1270 self.commitToPath = commit

1271

1272 def setFilename(self,f) :

1273 self.inputState.filename = f

1274

1275 def setLine(self,line) :

1276 self.inputState.line = line

1277

1278 def setText(self,s) :

1279 self.resetText()

1280 self.text.append(s)

1281

1282 def getCaseSensitive(self) :

1283 return self.caseSensitive

1284

1285 def getCaseSensitiveLiterals(self) :

1286 return self.caseSensitiveLiterals

1287

1288 def getColumn(self) :

1289 return self.inputState.column

1290

1291 def setColumn(self,c) :

1292 self.inputState.column = c

1293

1294 def getCommitToPath(self) :

1295 return self.commitToPath

1296

1297 def getFilename(self) :

1298 return self.inputState.filename

1299

1300 def getInputBuffer(self) :

1301 return self.inputState.input

1302

1303 def getInputState(self) :

1304 return self.inputState

1305

1306 def setInputState(self,state) :

1307 assert isinstance(state,LexerSharedInputState)

1308 self.inputState = state

1309

1310 def getLine(self) :

1311 return self.inputState.line

1312

1313 def getText(self) :

1314 return str(self.text)

1315

1316 def getTokenObject(self) :

1317 return self._returnToken

1318

1319 def LA(self,i) :

1320 c = self.inputState.input.LA(i)

1321 if not self.caseSensitive:

1322 ### E0006

1323 c = c.__class__.lower(c)

1324 return c

1325

1326 def makeToken(self,type) :

1327 try:

1328 ## dynamically load a class

1329 assert self.tokenClass

1330 tok = self.tokenClass()

1331 tok.setType(type)

1332 tok.setColumn(self.inputState.tokenStartColumn)

1333 tok.setLine(self.inputState.tokenStartLine)

1334 return tok

1335 except:

1336 self.panic("unable to create new token")

1337 return Token.badToken

1338

1339 def mark(self) :

1340 return self.inputState.input.mark()

1341

1342 def _match_bitset(self,b) :

1343 if b.member(self.LA(1)):

1344 self.consume()

1345 else:

1346 raise MismatchedCharException(self.LA(1), b, False, self)

1347

1348 def _match_string(self,s) :

1349 for c in s:

1350 if self.LA(1) == c:

1351 self.consume()

1352 else:

1353 raise MismatchedCharException(self.LA(1), c, False, self)

1354

1355 def match(self,item):

1356 if is_string_type(item):

1357 return self._match_string(item)

1358 else:

1359 return self._match_bitset(item)

1360

1361 def matchNot(self,c) :

1362 if self.LA(1) != c:

1363 self.consume()

1364 else:

1365 raise MismatchedCharException(self.LA(1), c, True, self)

1366

1367 def matchRange(self,c1,c2) :

1368 if self.LA(1) < c1 or self.LA(1) > c2 :

1369 raise MismatchedCharException(self.LA(1), c1, c2, False, self)

1370 else:

1371 self.consume()

1372

1373 def newline(self) :

1374 self.inputState.line += 1

1375 self.inputState.column = 1

1376

1377 def tab(self) :

1378 c = self.getColumn()

1379 nc = ( ((c-1)/self.tabsize) + 1) * self.tabsize + 1

1380 self.setColumn(nc)

1381

1382 def panic(self,s='') :

1383 print("CharScanner: panic: " + s)

1384 sys.exit(1)

1385

1386 def reportError(self,s) :

1387 if not self.getFilename():

1388 print("error: " + str(s))

1389 else:

1390 print(self.getFilename() + ": error: " + str(s))

1391

1392 def reportWarning(self,s) :

1393 if not self.getFilename():

1394 print("warning: " + str(s))

1395 else:

1396 print(self.getFilename() + ": warning: " + str(s))

1397

1398 def resetText(self) :

1399 self.text.setLength(0)

1400 self.inputState.tokenStartColumn = self.inputState.column

1401 self.inputState.tokenStartLine = self.inputState.line

1402

1403 def rewind(self,pos) :

1404 self.inputState.input.rewind(pos)

1405

1406 def setTokenObjectClass(self,cl):

1407 self.tokenClass = cl

1408

1409 def testForLiteral(self,token):

1410 if not token:

1411 return

1412 assert isinstance(token,Token)

1413

1414 _type = token.getType()

1415

1416 ## special tokens can't be literals

1417 if _type in [SKIP,INVALID_TYPE,EOF_TYPE,NULL_TREE_LOOKAHEAD] :

1418 return

1419

1420 _text = token.getText()

1421 if not _text:

1422 return

1423

1424 assert is_string_type(_text)

1425 _type = self.testLiteralsTable(_text,_type)

1426 token.setType(_type)

1427 return _type

1428

1429 def testLiteralsTable(self,*args):

1430 if is_string_type(args[0]):

1431 s = args[0]

1432 i = args[1]

1433 else:

1434 s = self.text.getString()

1435 i = args[0]

1436

1437 ## check whether integer has been given

1438 if not isinstance(i,int):

1439 assert isinstance(i,int)

1440

1441 ## check whether we have a dict

1442 assert isinstance(self.literals,dict)

1443 try:

1444 ## E0010

1445 if not self.caseSensitiveLiterals:

1446 s = s.__class__.lower(s)

1447 i = self.literals[s]

1448 except:

1449 pass

1450 return i

1451

1452 def toLower(self,c):

1453 return c.__class__.lower()

1454

1455 def traceIndent(self):

1456 print(' ' * self.traceDepth)

1457

1458 def traceIn(self,rname):

1459 self.traceDepth += 1

1460 self.traceIndent()

1461 print("> lexer %s c== %s" % (rname,self.LA(1)))

1462

1463 def traceOut(self,rname):

1464 self.traceIndent()

1465 print("< lexer %s c== %s" % (rname,self.LA(1)))

1466 self.traceDepth -= 1

1467

1468 def uponEOF(self):

1469 pass

1470

1471 def append(self,c):

1472 if self.saveConsumedInput :

1473 self.text.append(c)

1474

1475 def commit(self):

1476 self.inputState.input.commit()

1477

1478 def consume(self):

1479 if not self.inputState.guessing:

1480 c = self.LA(1)

1481 if self.caseSensitive:

1482 self.append(c)

1483 else:

1484 # use input.LA(), not LA(), to get original case

1485 # CharScanner.LA() would toLower it.

1486 c = self.inputState.input.LA(1)

1487 self.append(c)

1488

1489 if c and c in "\t":

1490 self.tab()

1491 else:

1492 self.inputState.column += 1

1493 self.inputState.input.consume()

1494

1495 ## Consume chars until one matches the given char

1496 def consumeUntil_char(self,c):

1497 while self.LA(1) != EOF_CHAR and self.LA(1) != c:

1498 self.consume()

1499

1500 ## Consume chars until one matches the given set

1501 def consumeUntil_bitset(self,bitset):

1502 while self.LA(1) != EOF_CHAR and not self.set.member(self.LA(1)):

1503 self.consume()

1504

1505 ### If symbol seen is EOF then generate and set token, otherwise

1506 ### throw exception.

1507 def default(self,la1):

1508 if not la1 :

1509 self.uponEOF()

1510 self._returnToken = self.makeToken(EOF_TYPE)

1511 else:

1512 self.raise_NoViableAlt(la1)

1513

1514 def filterdefault(self,la1,*args):

1515 if not la1:

1516 self.uponEOF()

1517 self._returnToken = self.makeToken(EOF_TYPE)

1518 return

1519

1520 if not args:

1521 self.consume()

1522 raise TryAgain()

1523 else:

1524 ### apply filter object

1525 self.commit();

1526 try:

1527 func=args[0]

1528 func(*args[1:])

1529 except RecognitionException as e:

1530 ## catastrophic failure

1531 self.reportError(e);

1532 self.consume();

1533 raise TryAgain()

1534

1535 def raise_NoViableAlt(self,la1=None):

1536 if not la1: la1 = self.LA(1)

1537 fname = self.getFilename()

1538 line = self.getLine()

1539 col = self.getColumn()

1540 raise NoViableAltForCharException(la1,fname,line,col)

1541

1542 def set_return_token(self,_create,_token,_ttype,_offset):

1543 if _create and not _token and (not _ttype == SKIP):

1544 string = self.text.getString(_offset)

1545 _token = self.makeToken(_ttype)

1546 _token.setText(string)

1547 self._returnToken = _token

1548 return _token

1549

1550###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1551### CharScannerIterator ###

1552###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1553

1554class CharScannerIterator:

1555

1556 def __init__(self,inst):

1557 if isinstance(inst,CharScanner):

1558 self.inst = inst

1559 return

1560 raise TypeError("CharScannerIterator requires CharScanner object")

1561

1562 def next(self):

1563 assert self.inst

1564 item = self.inst.nextToken()

1565 if not item or item.isEOF():

1566 raise StopIteration()

1567 return item

1568

1569###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1570### BitSet ###

1571###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1572

1573### I'm assuming here that a long is 64bits. It appears however, that

1574### a long is of any size. That means we can use a single long as the

1575### bitset (!), ie. Python would do almost all the work (TBD).

1576

1577class BitSet(object):

1578 BITS = 64

1579 NIBBLE = 4

1580 LOG_BITS = 6

1581 MOD_MASK = BITS -1

1582

1583 def __init__(self,data=None):

1584 if not data: 1584 ↛ 1585line 1584 didn't jump to line 1585, because the condition on line 1584 was never true

1585 BitSet.__init__(self,[long(0)])

1586 return

1587 if isinstance(data,int): 1587 ↛ 1588line 1587 didn't jump to line 1588, because the condition on line 1587 was never true

1588 BitSet.__init__(self,[long(data)])

1589 return

1590 if isinstance(data,long): 1590 ↛ 1591line 1590 didn't jump to line 1591, because the condition on line 1590 was never true

1591 BitSet.__init__(self,[data])

1592 return

1593 if not isinstance(data,list): 1593 ↛ 1594line 1593 didn't jump to line 1594, because the condition on line 1593 was never true

1594 raise TypeError("BitSet requires integer, long, or " +

1595 "list argument")

1596 for x in data:

1597 if not isinstance(x, int_types): 1597 ↛ 1598line 1597 didn't jump to line 1598, because the condition on line 1597 was never true

1598 raise TypeError(self,"List argument item is " +

1599 "not a long: %s" % (x))

1600 self.data = data

1601

1602 def __str__(self):

1603 bits = len(self.data) * BitSet.BITS

1604 s = ""

1605 for i in xrange(0,bits):

1606 if self.at(i):

1607 s += "1"

1608 else:

1609 s += "o"

1610 if not ((i+1) % 10):

1611 s += '|%s|' % (i+1)

1612 return s

1613

1614 def __repr__(self):

1615 return str(self)

1616

1617 def member(self,item):

1618 if not item:

1619 return False

1620

1621 if isinstance(item,int):

1622 return self.at(item)

1623

1624 if not is_string_type(item):

1625 raise TypeError(self,"char or unichar expected: %s" % (item))

1626

1627 ## char is a (unicode) string with at most lenght 1, ie.

1628 ## a char.

1629

1630 if len(item) != 1:

1631 raise TypeError(self,"char expected: %s" % (item))

1632

1633 ### handle ASCII/UNICODE char

1634 num = ord(item)

1635

1636 ### check whether position num is in bitset

1637 return self.at(num)

1638

1639 def wordNumber(self,bit):

1640 return bit >> BitSet.LOG_BITS

1641

1642 def bitMask(self,bit):

1643 pos = bit & BitSet.MOD_MASK ## bit mod BITS

1644 return (1 << pos)

1645

1646 def set(self,bit,on=True):

1647 # grow bitset as required (use with care!)

1648 i = self.wordNumber(bit)

1649 mask = self.bitMask(bit)

1650 if i>=len(self.data):

1651 d = i - len(self.data) + 1

1652 for x in xrange(0,d):

1653 self.data.append(0)

1654 assert len(self.data) == i+1

1655 if on:

1656 self.data[i] |= mask

1657 else:

1658 self.data[i] &= (~mask)

1659

1660 ### make add an alias for set

1661 add = set

1662

1663 def off(self,bit,off=True):

1664 self.set(bit,not off)

1665

1666 def at(self,bit):

1667 i = self.wordNumber(bit)

1668 v = self.data[i]

1669 m = self.bitMask(bit)

1670 return v & m

1671

1672

1673###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1674### some further funcs ###

1675###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1676

1677def illegalarg_ex(func):

1678 raise ValueError(

1679 "%s is only valid if parser is built for debugging" %

1680 (func.func_name))

1681

1682def runtime_ex(func):

1683 raise RuntimeError(

1684 "%s is only valid if parser is built for debugging" %

1685 (func.func_name))

1686

1687###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1688### TokenBuffer ###

1689###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1690

1691class TokenBuffer(object):

1692 def __init__(self,stream):

1693 self.input = stream

1694 self.nMarkers = 0

1695 self.markerOffset = 0

1696 self.numToConsume = 0

1697 self.queue = Queue()

1698

1699 def reset(self) :

1700 self.nMarkers = 0

1701 self.markerOffset = 0

1702 self.numToConsume = 0

1703 self.queue.reset()

1704

1705 def consume(self) :

1706 self.numToConsume += 1

1707

1708 def fill(self, amount):

1709 self.syncConsume()

1710 while self.queue.length() < (amount + self.markerOffset):

1711 self.queue.append(self.input.nextToken())

1712

1713 def getInput(self):

1714 return self.input

1715

1716 def LA(self,k) :

1717 self.fill(k)

1718 return self.queue.elementAt(self.markerOffset + k - 1).type

1719

1720 def LT(self,k) :

1721 self.fill(k)

1722 return self.queue.elementAt(self.markerOffset + k - 1)

1723

1724 def mark(self) :

1725 self.syncConsume()

1726 self.nMarkers += 1

1727 return self.markerOffset

1728

1729 def rewind(self,mark) :

1730 self.syncConsume()

1731 self.markerOffset = mark

1732 self.nMarkers -= 1

1733

1734 def syncConsume(self) :

1735 while self.numToConsume > 0:

1736 if self.nMarkers > 0:

1737 # guess mode -- leave leading characters and bump offset.

1738 self.markerOffset += 1

1739 else:

1740 # normal mode -- remove first character

1741 self.queue.removeFirst()

1742 self.numToConsume -= 1

1743

1744 def __str__(self):

1745 return "(%s,%s,%s,%s,%s)" % (

1746 self.input,

1747 self.nMarkers,

1748 self.markerOffset,

1749 self.numToConsume,

1750 self.queue)

1751

1752 def __repr__(self):

1753 return str(self)

1754

1755###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1756### ParserSharedInputState ###

1757###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1758

1759class ParserSharedInputState(object):

1760

1761 def __init__(self):

1762 self.input = None

1763 self.reset()

1764

1765 def reset(self):

1766 self.guessing = 0

1767 self.filename = None

1768 if self.input:

1769 self.input.reset()

1770

1771###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1772### Parser ###

1773###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

1774

1775class Parser(object):

1776

1777 def __init__(self, *args, **kwargs):

1778 self.tokenNames = None

1779 self.returnAST = None

1780 self.astFactory = None

1781 self.tokenTypeToASTClassMap = {}

1782 self.ignoreInvalidDebugCalls = False

1783 self.traceDepth = 0

1784 if not args:

1785 self.inputState = ParserSharedInputState()

1786 return

1787 arg0 = args[0]

1788 assert isinstance(arg0,ParserSharedInputState)

1789 self.inputState = arg0

1790 return

1791

1792 def getTokenTypeToASTClassMap(self):

1793 return self.tokenTypeToASTClassMap

1794

1795

1796 def addMessageListener(self, l):

1797 if not self.ignoreInvalidDebugCalls:

1798 illegalarg_ex(self.addMessageListener)

1799

1800 def addParserListener(self,l) :

1801 if (not self.ignoreInvalidDebugCalls) :

1802 illegalarg_ex(self.addParserListener)

1803

1804 def addParserMatchListener(self, l) :

1805 if (not self.ignoreInvalidDebugCalls) :

1806 illegalarg_ex(self.addParserMatchListener)

1807

1808 def addParserTokenListener(self, l) :

1809 if (not self.ignoreInvalidDebugCalls):

1810 illegalarg_ex(self.addParserTokenListener)

1811

1812 def addSemanticPredicateListener(self, l) :

1813 if (not self.ignoreInvalidDebugCalls):

1814 illegalarg_ex(self.addSemanticPredicateListener)

1815

1816 def addSyntacticPredicateListener(self, l) :

1817 if (not self.ignoreInvalidDebugCalls):

1818 illegalarg_ex(self.addSyntacticPredicateListener)

1819

1820 def addTraceListener(self, l) :

1821 if (not self.ignoreInvalidDebugCalls):

1822 illegalarg_ex(self.addTraceListener)

1823

1824 def consume(self):

1825 raise NotImplementedError()

1826

1827 def _consumeUntil_type(self,tokenType):

1828 while self.LA(1) != EOF_TYPE and self.LA(1) != tokenType:

1829 self.consume()

1830

1831 def _consumeUntil_bitset(self, set):

1832 while self.LA(1) != EOF_TYPE and not set.member(self.LA(1)):

1833 self.consume()

1834

1835 def consumeUntil(self,arg):

1836 if isinstance(arg,int):

1837 self._consumeUntil_type(arg)

1838 else:

1839 self._consumeUntil_bitset(arg)

1840

1841 def defaultDebuggingSetup(self):

1842 pass

1843

1844 def getAST(self) :

1845 return self.returnAST

1846

1847 def getASTFactory(self) :

1848 return self.astFactory

1849

1850 def getFilename(self) :

1851 return self.inputState.filename

1852

1853 def getInputState(self) :

1854 return self.inputState

1855

1856 def setInputState(self, state) :

1857 self.inputState = state

1858

1859 def getTokenName(self,num) :

1860 return self.tokenNames[num]

1861

1862 def getTokenNames(self) :

1863 return self.tokenNames

1864

1865 def isDebugMode(self) :

1866 return self.false

1867

1868 def LA(self, i):

1869 raise NotImplementedError()

1870

1871 def LT(self, i):

1872 raise NotImplementedError()

1873

1874 def mark(self):

1875 return self.inputState.input.mark()

1876

1877 def _match_int(self,t):

1878 if (self.LA(1) != t):

1879 raise MismatchedTokenException(

1880 self.tokenNames, self.LT(1), t, False, self.getFilename())

1881 else:

1882 self.consume()

1883

1884 def _match_set(self, b):

1885 if (not b.member(self.LA(1))):

1886 raise MismatchedTokenException(

1887 self.tokenNames,self.LT(1), b, False, self.getFilename())

1888 else:

1889 self.consume()

1890

1891 def match(self,set) :

1892 if isinstance(set,int):

1893 self._match_int(set)

1894 return

1895 if isinstance(set,BitSet):

1896 self._match_set(set)

1897 return

1898 raise TypeError("Parser.match requires integer ot BitSet argument")

1899

1900 def matchNot(self,t):

1901 if self.LA(1) == t:

1902 raise MismatchedTokenException(

1903 self.tokenNames, self.LT(1), t, True, self.getFilename())

1904 else:

1905 self.consume()

1906

1907 def removeMessageListener(self, l) :

1908 if (not self.ignoreInvalidDebugCalls):

1909 runtime_ex(self.removeMessageListener)

1910

1911 def removeParserListener(self, l) :

1912 if (not self.ignoreInvalidDebugCalls):

1913 runtime_ex(self.removeParserListener)

1914

1915 def removeParserMatchListener(self, l) :

1916 if (not self.ignoreInvalidDebugCalls):

1917 runtime_ex(self.removeParserMatchListener)

1918

1919 def removeParserTokenListener(self, l) :

1920 if (not self.ignoreInvalidDebugCalls):

1921 runtime_ex(self.removeParserTokenListener)

1922

1923 def removeSemanticPredicateListener(self, l) :

1924 if (not self.ignoreInvalidDebugCalls):

1925 runtime_ex(self.removeSemanticPredicateListener)

1926

1927 def removeSyntacticPredicateListener(self, l) :

1928 if (not self.ignoreInvalidDebugCalls):

1929 runtime_ex(self.removeSyntacticPredicateListener)

1930

1931 def removeTraceListener(self, l) :

1932 if (not self.ignoreInvalidDebugCalls):

1933 runtime_ex(self.removeTraceListener)

1934

1935 def reportError(self,x) :

1936 fmt = "syntax error:"

1937 f = self.getFilename()

1938 if f:

1939 fmt = ("%s:" % f) + fmt

1940 if isinstance(x,Token):

1941 line = x.getColumn()

1942 col = x.getLine()

1943 text = x.getText()

1944 fmt = fmt + 'unexpected symbol at line %s (column %s) : "%s"'

1945 print(fmt % (line,col,text), file=sys.stderr)

1946 else:

1947 print(fmt,str(x), file=sys.stderr)

1948

1949 def reportWarning(self,s):

1950 f = self.getFilename()

1951 if f:

1952 print("%s:warning: %s" % (f,str(s)))

1953 else:

1954 print("warning: %s" % (str(s)))

1955

1956 def rewind(self, pos) :

1957 self.inputState.input.rewind(pos)

1958

1959 def setASTFactory(self, f) :

1960 self.astFactory = f

1961

1962 def setASTNodeClass(self, cl) :

1963 self.astFactory.setASTNodeType(cl)

1964

1965 def setASTNodeType(self, nodeType) :

1966 self.setASTNodeClass(nodeType)

1967

1968 def setDebugMode(self, debugMode) :

1969 if (not self.ignoreInvalidDebugCalls):

1970 runtime_ex(self.setDebugMode)

1971

1972 def setFilename(self, f) :

1973 self.inputState.filename = f

1974

1975 def setIgnoreInvalidDebugCalls(self, value) :

1976 self.ignoreInvalidDebugCalls = value

1977

1978 def setTokenBuffer(self, t) :

1979 self.inputState.input = t

1980

1981 def traceIndent(self):

1982 print(" " * self.traceDepth)

1983

1984 def traceIn(self,rname):

1985 self.traceDepth += 1

1986 self.trace("> ", rname)

1987

1988 def traceOut(self,rname):

1989 self.trace("< ", rname)

1990 self.traceDepth -= 1

1991

1992 ### wh: moved from ASTFactory to Parser

1993 def addASTChild(self,currentAST, child):

1994 if not child:

1995 return

1996 if not currentAST.root:

1997 currentAST.root = child

1998 elif not currentAST.child:

1999 currentAST.root.setFirstChild(child)

2000 else:

2001 currentAST.child.setNextSibling(child)

2002 currentAST.child = child

2003 currentAST.advanceChildToEnd()

2004

2005 ### wh: moved from ASTFactory to Parser

2006 def makeASTRoot(self,currentAST,root) :

2007 if root:

2008 ### Add the current root as a child of new root

2009 root.addChild(currentAST.root)

2010 ### The new current child is the last sibling of the old root

2011 currentAST.child = currentAST.root

2012 currentAST.advanceChildToEnd()

2013 ### Set the new root

2014 currentAST.root = root

2015

2016###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2017### LLkParser ###

2018###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2019

2020class LLkParser(Parser):

2021

2022 def __init__(self, *args, **kwargs):

2023 try:

2024 arg1 = args[0]

2025 except:

2026 arg1 = 1

2027

2028 if isinstance(arg1,int):

2029 super(LLkParser,self).__init__()

2030 self.k = arg1

2031 return

2032

2033 if isinstance(arg1,ParserSharedInputState):

2034 super(LLkParser,self).__init__(arg1)

2035 self.set_k(1,*args)

2036 return

2037

2038 if isinstance(arg1,TokenBuffer):

2039 super(LLkParser,self).__init__()

2040 self.setTokenBuffer(arg1)

2041 self.set_k(1,*args)

2042 return

2043

2044 if isinstance(arg1,TokenStream):

2045 super(LLkParser,self).__init__()

2046 tokenBuf = TokenBuffer(arg1)

2047 self.setTokenBuffer(tokenBuf)

2048 self.set_k(1,*args)

2049 return

2050

2051 ### unknown argument

2052 raise TypeError("LLkParser requires integer, " +

2053 "ParserSharedInputStream or TokenStream argument")

2054

2055 def consume(self):

2056 self.inputState.input.consume()

2057

2058 def LA(self,i):

2059 return self.inputState.input.LA(i)

2060

2061 def LT(self,i):

2062 return self.inputState.input.LT(i)

2063

2064 def set_k(self,index,*args):

2065 try:

2066 self.k = args[index]

2067 except:

2068 self.k = 1

2069

2070 def trace(self,ee,rname):

2071 print(type(self))

2072 self.traceIndent()

2073 guess = ""

2074 if self.inputState.guessing > 0:

2075 guess = " [guessing]"

2076 print((ee + rname + guess))

2077 for i in xrange(1,self.k+1):

2078 if i != 1:

2079 print(", ")

2080 if self.LT(i) :

2081 v = self.LT(i).getText()

2082 else:

2083 v = "null"

2084 print("LA(%s) == %s" % (i,v))

2085 print("\n")

2086

2087 def traceIn(self,rname):

2088 self.traceDepth += 1;

2089 self.trace("> ", rname);

2090

2091 def traceOut(self,rname):

2092 self.trace("< ", rname);

2093 self.traceDepth -= 1;

2094

2095###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2096### TreeParserSharedInputState ###

2097###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2098

2099class TreeParserSharedInputState(object):

2100 def __init__(self):

2101 self.guessing = 0

2102

2103###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2104### TreeParser ###

2105###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2106

2107class TreeParser(object):

2108

2109 def __init__(self, *args, **kwargs):

2110 self.inputState = TreeParserSharedInputState()

2111 self._retTree = None

2112 self.tokenNames = []

2113 self.returnAST = None

2114 self.astFactory = ASTFactory()

2115 self.traceDepth = 0

2116

2117 def getAST(self):

2118 return self.returnAST

2119

2120 def getASTFactory(self):

2121 return self.astFactory

2122

2123 def getTokenName(self,num) :

2124 return self.tokenNames[num]

2125

2126 def getTokenNames(self):

2127 return self.tokenNames

2128

2129 def match(self,t,set) :

2130 assert isinstance(set,int) or isinstance(set,BitSet)

2131 if not t or t == ASTNULL:

2132 raise MismatchedTokenException(self.getTokenNames(), t,set, False)

2133

2134 if isinstance(set,int) and t.getType() != set:

2135 raise MismatchedTokenException(self.getTokenNames(), t,set, False)

2136

2137 if isinstance(set,BitSet) and not set.member(t.getType):

2138 raise MismatchedTokenException(self.getTokenNames(), t,set, False)

2139

2140 def matchNot(self,t, ttype) :

2141 if not t or (t == ASTNULL) or (t.getType() == ttype):

2142 raise MismatchedTokenException(self.getTokenNames(), t, ttype, True)

2143

2144 def reportError(self,ex):

2145 print("error:",ex, file=sys.stderr)

2146

2147 def reportWarning(self, s):

2148 print("warning:",s)

2149

2150 def setASTFactory(self,f):

2151 self.astFactory = f

2152

2153 def setASTNodeType(self,nodeType):

2154 self.setASTNodeClass(nodeType)

2155

2156 def setASTNodeClass(self,nodeType):

2157 self.astFactory.setASTNodeType(nodeType)

2158

2159 def traceIndent(self):

2160 print(" " * self.traceDepth)

2161

2162 def traceIn(self,rname,t):

2163 self.traceDepth += 1

2164 self.traceIndent()

2165 print(("> " + rname + "(" +

2166 ifelse(t,str(t),"null") + ")" +

2167 ifelse(self.inputState.guessing>0,"[guessing]","")))

2168

2169 def traceOut(self,rname,t):

2170 self.traceIndent()

2171 print(("< " + rname + "(" +

2172 ifelse(t,str(t),"null") + ")" +

2173 ifelse(self.inputState.guessing>0,"[guessing]","")))

2174 self.traceDepth -= 1

2175

2176 ### wh: moved from ASTFactory to TreeParser

2177 def addASTChild(self,currentAST, child):

2178 if not child:

2179 return

2180 if not currentAST.root:

2181 currentAST.root = child

2182 elif not currentAST.child:

2183 currentAST.root.setFirstChild(child)

2184 else:

2185 currentAST.child.setNextSibling(child)

2186 currentAST.child = child

2187 currentAST.advanceChildToEnd()

2188

2189 ### wh: moved from ASTFactory to TreeParser

2190 def makeASTRoot(self,currentAST,root):

2191 if root:

2192 ### Add the current root as a child of new root

2193 root.addChild(currentAST.root)

2194 ### The new current child is the last sibling of the old root

2195 currentAST.child = currentAST.root

2196 currentAST.advanceChildToEnd()

2197 ### Set the new root

2198 currentAST.root = root

2199

2200###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2201### funcs to work on trees ###

2202###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2203

2204def rightmost(ast):

2205 if ast:

2206 while(ast.right):

2207 ast = ast.right

2208 return ast

2209

2210def cmptree(s,t,partial):

2211 while(s and t):

2212 ### as a quick optimization, check roots first.

2213 if not s.equals(t):

2214 return False

2215

2216 ### if roots match, do full list match test on children.

2217 if not cmptree(s.getFirstChild(),t.getFirstChild(),partial):

2218 return False

2219

2220 s = s.getNextSibling()

2221 t = t.getNextSibling()

2222

2223 r = ifelse(partial,not t,not s and not t)

2224 return r

2225

2226###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2227### AST ###

2228###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2229

2230class AST(object):

2231 def __init__(self):

2232 pass

2233

2234 def addChild(self, c):

2235 pass

2236

2237 def equals(self, t):

2238 return False

2239

2240 def equalsList(self, t):

2241 return False

2242

2243 def equalsListPartial(self, t):

2244 return False

2245

2246 def equalsTree(self, t):

2247 return False

2248

2249 def equalsTreePartial(self, t):

2250 return False

2251

2252 def findAll(self, tree):

2253 return None

2254

2255 def findAllPartial(self, subtree):

2256 return None

2257

2258 def getFirstChild(self):

2259 return self

2260

2261 def getNextSibling(self):

2262 return self

2263

2264 def getText(self):

2265 return ""

2266

2267 def getType(self):

2268 return INVALID_TYPE

2269

2270 def getLine(self):

2271 return 0

2272

2273 def getColumn(self):

2274 return 0

2275

2276 def getNumberOfChildren(self):

2277 return 0

2278

2279 def initialize(self, t):

2280 pass

2281

2282 def setFirstChild(self, c):

2283 pass

2284

2285 def setNextSibling(self, n):

2286 pass

2287

2288 def setText(self, text):

2289 pass

2290

2291 def setType(self, ttype):

2292 pass

2293

2294 def toString(self):

2295 self.getText()

2296

2297 __str__ = toString

2298

2299 def toStringList(self):

2300 return self.getText()

2301

2302 def toStringTree(self):

2303 return self.getText()

2304

2305###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2306### ASTNULLType ###

2307###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2308

2309### There is only one instance of this class **/

2310class ASTNULLType(AST):

2311 def __init__(self):

2312 AST.__init__(self)

2313 pass

2314

2315 def getText(self):

2316 return "<ASTNULL>"

2317

2318 def getType(self):

2319 return NULL_TREE_LOOKAHEAD

2320

2321

2322###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2323### BaseAST ###

2324###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2325

2326class BaseAST(AST):

2327

2328 verboseStringConversion = False

2329 tokenNames = None

2330

2331 def __init__(self):

2332 self.down = None ## kid

2333 self.right = None ## sibling

2334

2335 def addChild(self,node):

2336 if node:

2337 t = rightmost(self.down)

2338 if t:

2339 t.right = node

2340 else:

2341 assert not self.down

2342 self.down = node

2343

2344 def getNumberOfChildren(self):

2345 t = self.down

2346 n = 0

2347 while t:

2348 n += 1

2349 t = t.right

2350 return n

2351

2352 def doWorkForFindAll(self,v,target,partialMatch):

2353 sibling = self

2354

2355 while sibling:

2356 c1 = partialMatch and sibling.equalsTreePartial(target)

2357 if c1:

2358 v.append(sibling)

2359 else:

2360 c2 = not partialMatch and sibling.equalsTree(target)

2361 if c2:

2362 v.append(sibling)

2363

2364 ### regardless of match or not, check any children for matches

2365 if sibling.getFirstChild():

2366 sibling.getFirstChild().doWorkForFindAll(v,target,partialMatch)

2367

2368 sibling = sibling.getNextSibling()

2369

2370 ### Is node t equal to 'self' in terms of token type and text?

2371 def equals(self,t):

2372 if not t:

2373 return False

2374 return self.getText() == t.getText() and self.getType() == t.getType()

2375

2376 ### Is t an exact structural and equals() match of this tree. The

2377 ### 'self' reference is considered the start of a sibling list.

2378 ###

2379 def equalsList(self, t):

2380 return cmptree(self, t, partial=False)

2381

2382 ### Is 't' a subtree of this list?

2383 ### The siblings of the root are NOT ignored.

2384 ###

2385 def equalsListPartial(self,t):

2386 return cmptree(self,t,partial=True)

2387

2388 ### Is tree rooted at 'self' equal to 't'? The siblings

2389 ### of 'self' are ignored.

2390 ###

2391 def equalsTree(self, t):

2392 return self.equals(t) and \

2393 cmptree(self.getFirstChild(), t.getFirstChild(), partial=False)

2394

2395 ### Is 't' a subtree of the tree rooted at 'self'? The siblings

2396 ### of 'self' are ignored.

2397 ###

2398 def equalsTreePartial(self, t):

2399 if not t:

2400 return True

2401 return self.equals(t) and cmptree(

2402 self.getFirstChild(), t.getFirstChild(), partial=True)

2403

2404 ### Walk the tree looking for all exact subtree matches. Return

2405 ### an ASTEnumerator that lets the caller walk the list

2406 ### of subtree roots found herein.

2407 def findAll(self,target):

2408 roots = []

2409

2410 ### the empty tree cannot result in an enumeration

2411 if not target:

2412 return None

2413 # find all matches recursively

2414 self.doWorkForFindAll(roots, target, False)

2415 return roots

2416

2417 ### Walk the tree looking for all subtrees. Return

2418 ### an ASTEnumerator that lets the caller walk the list

2419 ### of subtree roots found herein.

2420 def findAllPartial(self,sub):

2421 roots = []

2422

2423 ### the empty tree cannot result in an enumeration

2424 if not sub:

2425 return None

2426

2427 self.doWorkForFindAll(roots, sub, True) ### find all matches recursively

2428 return roots

2429

2430 ### Get the first child of this node None if not children

2431 def getFirstChild(self):

2432 return self.down

2433

2434 ### Get the next sibling in line after this one

2435 def getNextSibling(self):

2436 return self.right

2437

2438 ### Get the token text for this node

2439 def getText(self):

2440 return ""

2441

2442 ### Get the token type for this node

2443 def getType(self):

2444 return 0

2445

2446 def getLine(self):

2447 return 0

2448

2449 def getColumn(self):

2450 return 0

2451

2452 ### Remove all children */

2453 def removeChildren(self):

2454 self.down = None

2455

2456 def setFirstChild(self,c):

2457 self.down = c

2458

2459 def setNextSibling(self, n):

2460 self.right = n

2461

2462 ### Set the token text for this node

2463 def setText(self, text):

2464 pass

2465

2466 ### Set the token type for this node

2467 def setType(self, ttype):

2468 pass

2469

2470 ### static

2471 def setVerboseStringConversion(verbose,names):

2472 verboseStringConversion = verbose

2473 tokenNames = names

2474 setVerboseStringConversion = staticmethod(setVerboseStringConversion)

2475

2476 ### Return an array of strings that maps token ID to it's text.

2477 ## @since 2.7.3

2478 def getTokenNames():

2479 return tokenNames

2480

2481 def toString(self):

2482 return self.getText()

2483

2484 ### return tree as lisp string - sibling included

2485 def toStringList(self):

2486 ts = self.toStringTree()

2487 sib = self.getNextSibling()

2488 if sib:

2489 ts += sib.toStringList()

2490 return ts

2491

2492 __str__ = toStringList

2493

2494 ### return tree as string - siblings ignored

2495 def toStringTree(self):

2496 ts = ""

2497 kid = self.getFirstChild()

2498 if kid:

2499 ts += " ("

2500 ts += " " + self.toString()

2501 if kid:

2502 ts += kid.toStringList()

2503 ts += " )"

2504 return ts

2505

2506###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2507### CommonAST ###

2508###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2509

2510### Common AST node implementation

2511class CommonAST(BaseAST):

2512 def __init__(self,token=None):

2513 super(CommonAST,self).__init__()

2514 self.ttype = INVALID_TYPE

2515 self.text = "<no text>"

2516 self.line = 0

2517 self.column= 0

2518 self.initialize(token)

2519 #assert self.text

2520

2521 ### Get the token text for this node

2522 def getText(self):

2523 return self.text

2524

2525 ### Get the token type for this node

2526 def getType(self):

2527 return self.ttype

2528

2529 ### Get the line for this node

2530 def getLine(self):

2531 return self.line

2532

2533 ### Get the column for this node

2534 def getColumn(self):

2535 return self.column

2536

2537 def initialize(self,*args):

2538 if not args:

2539 return

2540

2541 arg0 = args[0]

2542

2543 if isinstance(arg0,int):

2544 arg1 = args[1]

2545 self.setType(arg0)

2546 self.setText(arg1)

2547 return

2548

2549 if isinstance(arg0,AST) or isinstance(arg0,Token):

2550 self.setText(arg0.getText())

2551 self.setType(arg0.getType())

2552 self.line = arg0.getLine()

2553 self.column = arg0.getColumn()

2554 return

2555

2556 ### Set the token text for this node

2557 def setText(self,text_):

2558 assert is_string_type(text_)

2559 self.text = text_

2560

2561 ### Set the token type for this node

2562 def setType(self,ttype_):

2563 assert isinstance(ttype_,int)

2564 self.ttype = ttype_

2565

2566###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2567### CommonASTWithHiddenTokens ###

2568###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2569

2570class CommonASTWithHiddenTokens(CommonAST):

2571

2572 def __init__(self,*args):

2573 CommonAST.__init__(self,*args)

2574 self.hiddenBefore = None

2575 self.hiddenAfter = None

2576

2577 def getHiddenAfter(self):

2578 return self.hiddenAfter

2579

2580 def getHiddenBefore(self):

2581 return self.hiddenBefore

2582

2583 def initialize(self,*args):

2584 CommonAST.initialize(self,*args)

2585 if args and isinstance(args[0],Token):

2586 assert isinstance(args[0],CommonHiddenStreamToken)

2587 self.hiddenBefore = args[0].getHiddenBefore()

2588 self.hiddenAfter = args[0].getHiddenAfter()

2589

2590###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2591### ASTPair ###

2592###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2593

2594class ASTPair(object):

2595 def __init__(self):

2596 self.root = None ### current root of tree

2597 self.child = None ### current child to which siblings are added

2598

2599 ### Make sure that child is the last sibling */

2600 def advanceChildToEnd(self):

2601 if self.child:

2602 while self.child.getNextSibling():

2603 self.child = self.child.getNextSibling()

2604

2605 ### Copy an ASTPair. Don't call it clone() because we want type-safety */

2606 def copy(self):

2607 tmp = ASTPair()

2608 tmp.root = self.root

2609 tmp.child = self.child

2610 return tmp

2611

2612 def toString(self):

2613 r = ifelse(not root,"null",self.root.getText())

2614 c = ifelse(not child,"null",self.child.getText())

2615 return "[%s,%s]" % (r,c)

2616

2617 __str__ = toString

2618 __repr__ = toString

2619

2620

2621###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2622### ASTFactory ###

2623###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2624

2625class ASTFactory(object):

2626 def __init__(self,table=None):

2627 self._class = None

2628 self._classmap = ifelse(table,table,None)

2629

2630 def create(self,*args):

2631 if not args:

2632 return self.create(INVALID_TYPE)

2633

2634 arg0 = args[0]

2635 arg1 = None

2636 arg2 = None

2637

2638 try:

2639 arg1 = args[1]

2640 arg2 = args[2]

2641 except:

2642 pass

2643

2644 # ctor(int)

2645 if isinstance(arg0,int) and not arg2:

2646 ### get class for 'self' type

2647 c = self.getASTNodeType(arg0)

2648 t = self.create(c)

2649 if t:

2650 t.initialize(arg0, ifelse(arg1,arg1,""))

2651 return t

2652

2653 # ctor(int,something)

2654 if isinstance(arg0,int) and arg2:

2655 t = self.create(arg2)

2656 if t:

2657 t.initialize(arg0,arg1)

2658 return t

2659

2660 # ctor(AST)

2661 if isinstance(arg0,AST):

2662 t = self.create(arg0.getType())

2663 if t:

2664 t.initialize(arg0)

2665 return t

2666

2667 # ctor(token)

2668 if isinstance(arg0,Token) and not arg1:

2669 ttype = arg0.getType()

2670 assert isinstance(ttype,int)

2671 t = self.create(ttype)

2672 if t:

2673 t.initialize(arg0)

2674 return t

2675

2676 # ctor(token,class)

2677 if isinstance(arg0,Token) and arg1:

2678 assert isinstance(arg1,type)

2679 assert issubclass(arg1,AST)

2680 # this creates instance of 'arg1' using 'arg0' as

2681 # argument. Wow, that's magic!

2682 t = arg1(arg0)

2683 assert t and isinstance(t,AST)

2684 return t

2685

2686 # ctor(class)

2687 if isinstance(arg0,type):

2688 ### next statement creates instance of type (!)

2689 t = arg0()

2690 assert isinstance(t,AST)

2691 return t

2692

2693

2694 def setASTNodeClass(self,className=None):

2695 if not className:

2696 return

2697 assert isinstance(className,type)

2698 assert issubclass(className,AST)

2699 self._class = className

2700

2701 ### kind of misnomer - use setASTNodeClass instead.

2702 setASTNodeType = setASTNodeClass

2703

2704 def getASTNodeClass(self):

2705 return self._class

2706

2707

2708

2709 def getTokenTypeToASTClassMap(self):

2710 return self._classmap

2711

2712 def setTokenTypeToASTClassMap(self,amap):

2713 self._classmap = amap

2714

2715 def error(self, e):

2716 import sys

2717 print(e, file=sys.stderr)

2718

2719 def setTokenTypeASTNodeType(self, tokenType, className):

2720 """

2721 Specify a mapping between a token type and a (AST) class.

2722 """

2723 if not self._classmap:

2724 self._classmap = {}

2725

2726 if not className:

2727 try:

2728 del self._classmap[tokenType]

2729 except:

2730 pass

2731 else:

2732 ### here we should also perform actions to ensure that

2733 ### a. class can be loaded

2734 ### b. class is a subclass of AST

2735 ###

2736 assert isinstance(className,type)

2737 assert issubclass(className,AST) ## a & b

2738 ### enter the class

2739 self._classmap[tokenType] = className

2740

2741 def getASTNodeType(self,tokenType):

2742 """

2743 For a given token type return the AST node type. First we

2744 lookup a mapping table, second we try _class

2745 and finally we resolve to "antlr.CommonAST".

2746 """

2747

2748 # first

2749 if self._classmap:

2750 try:

2751 c = self._classmap[tokenType]

2752 if c:

2753 return c

2754 except:

2755 pass

2756 # second

2757 if self._class:

2758 return self._class

2759

2760 # default

2761 return CommonAST

2762

2763 ### methods that have been moved to file scope - just listed

2764 ### here to be somewhat consistent with original API

2765 def dup(self,t):

2766 return dup(t,self)

2767

2768 def dupList(self,t):

2769 return dupList(t,self)

2770

2771 def dupTree(self,t):

2772 return dupTree(t,self)

2773

2774 ### methods moved to other classes

2775 ### 1. makeASTRoot -> Parser

2776 ### 2. addASTChild -> Parser

2777

2778 ### non-standard: create alias for longish method name

2779 maptype = setTokenTypeASTNodeType

2780

2781###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2782### ASTVisitor ###

2783###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2784

2785class ASTVisitor(object):

2786 def __init__(self,*args):

2787 pass

2788

2789 def visit(self,ast):

2790 pass

2791

2792###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2793### static methods and variables ###

2794###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx###

2795

2796ASTNULL = ASTNULLType()

2797

2798### wh: moved from ASTFactory as there's nothing ASTFactory-specific

2799### in this method.

2800def make(*nodes):

2801 if not nodes:

2802 return None

2803

2804 for i in xrange(0,len(nodes)):

2805 node = nodes[i]

2806 if node:

2807 assert isinstance(node,AST)

2808

2809 root = nodes[0]

2810 tail = None

2811 if root:

2812 root.setFirstChild(None)

2813

2814 for i in xrange(1,len(nodes)):

2815 if not nodes[i]:

2816 continue

2817 if not root:

2818 root = tail = nodes[i]

2819 elif not tail:

2820 root.setFirstChild(nodes[i])

2821 tail = root.getFirstChild()

2822 else:

2823 tail.setNextSibling(nodes[i])

2824 tail = tail.getNextSibling()

2825

2826 ### Chase tail to last sibling

2827 while tail.getNextSibling():

2828 tail = tail.getNextSibling()

2829 return root

2830

2831def dup(t,factory):

2832 if not t:

2833 return None

2834

2835 if factory:

2836 dup_t = factory.create(t.__class__)

2837 else:

2838 raise TypeError("dup function requires ASTFactory argument")

2839 dup_t.initialize(t)

2840 return dup_t

2841

2842def dupList(t,factory):

2843 result = dupTree(t,factory)

2844 nt = result

2845 while t:

2846 ## for each sibling of the root

2847 t = t.getNextSibling()

2848 nt.setNextSibling(dupTree(t,factory))

2849 nt = nt.getNextSibling()

2850 return result

2851

2852def dupTree(t,factory):

2853 result = dup(t,factory)

2854 if t:

2855 result.setFirstChild(dupList(t.getFirstChild(),factory))

2856 return result

2857

2858###xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

2859### $Id$

2860

2861# Local Variables: ***

2862# mode: python ***

2863# py-indent-offset: 4 ***

2864# End: ***