Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/sqlparse/engine/grouping.py: 12%

1# 

2# Copyright (C) 2009-2020 the sqlparse authors and contributors 

3# <see AUTHORS file> 

4# 

5# This module is part of python-sqlparse and is released under 

6# the BSD License: https://opensource.org/licenses/BSD-3-Clause 

7 

8from sqlparse import sql 

9from sqlparse import tokens as T 

10from sqlparse.utils import recurse, imt 

11 

12T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float) 

13T_STRING = (T.String, T.String.Single, T.String.Symbol) 

14T_NAME = (T.Name, T.Name.Placeholder) 

15 

16 

17def _group_matching(tlist, cls): 

18 """Groups Tokens that have beginning and end.""" 

19 opens = [] 

20 tidx_offset = 0 

21 for idx, token in enumerate(list(tlist)): 

22 tidx = idx - tidx_offset 

23 

24 if token.is_whitespace: 

25 # ~50% of tokens will be whitespace. Will checking early 

26 # for them avoid 3 comparisons, but then add 1 more comparison 

27 # for the other ~50% of tokens... 

28 continue 

29 

30 if token.is_group and not isinstance(token, cls): 

31 # Check inside previously grouped (i.e. parenthesis) if group 

32 # of different type is inside (i.e., case). though ideally should 

33 # should check for all open/close tokens at once to avoid recursion 

34 _group_matching(token, cls) 

35 continue 

36 

37 if token.match(*cls.M_OPEN): 

38 opens.append(tidx) 

39 

40 elif token.match(*cls.M_CLOSE): 

41 try: 

42 open_idx = opens.pop() 

43 except IndexError: 

44 # this indicates invalid sql and unbalanced tokens. 

45 # instead of break, continue in case other "valid" groups exist 

46 continue 

47 close_idx = tidx 

48 tlist.group_tokens(cls, open_idx, close_idx) 

49 tidx_offset += close_idx - open_idx 

50 

51 

52def group_brackets(tlist): 

53 _group_matching(tlist, sql.SquareBrackets) 

54 

55 

56def group_parenthesis(tlist): 

57 _group_matching(tlist, sql.Parenthesis) 

58 

59 

60def group_case(tlist): 

61 _group_matching(tlist, sql.Case) 

62 

63 

64def group_if(tlist): 

65 _group_matching(tlist, sql.If) 

66 

67 

68def group_for(tlist): 

69 _group_matching(tlist, sql.For) 

70 

71 

72def group_begin(tlist): 

73 _group_matching(tlist, sql.Begin) 

74 

75 

76def group_typecasts(tlist): 

77 def match(token): 

78 return token.match(T.Punctuation, '::') 

79 

80 def valid(token): 

81 return token is not None 

82 

83 def post(tlist, pidx, tidx, nidx): 

84 return pidx, nidx 

85 

86 valid_prev = valid_next = valid 

87 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

88 

89 

90def group_tzcasts(tlist): 

91 def match(token): 

92 return token.ttype == T.Keyword.TZCast 

93 

94 def valid_prev(token): 

95 return token is not None 

96 

97 def valid_next(token): 

98 return token is not None and ( 

99 token.is_whitespace 

100 or token.match(T.Keyword, 'AS') 

101 or token.match(*sql.TypedLiteral.M_CLOSE) 

102 ) 

103 

104 def post(tlist, pidx, tidx, nidx): 

105 return pidx, nidx 

106 

107 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

108 

109 

110def group_typed_literal(tlist): 

111 # definitely not complete, see e.g.: 

112 # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literal-syntax 

113 # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literals 

114 # https://www.postgresql.org/docs/9.1/datatype-datetime.html 

115 # https://www.postgresql.org/docs/9.1/functions-datetime.html 

116 def match(token): 

117 return imt(token, m=sql.TypedLiteral.M_OPEN) 

118 

119 def match_to_extend(token): 

120 return isinstance(token, sql.TypedLiteral) 

121 

122 def valid_prev(token): 

123 return token is not None 

124 

125 def valid_next(token): 

126 return token is not None and token.match(*sql.TypedLiteral.M_CLOSE) 

127 

128 def valid_final(token): 

129 return token is not None and token.match(*sql.TypedLiteral.M_EXTEND) 

130 

131 def post(tlist, pidx, tidx, nidx): 

132 return tidx, nidx 

133 

134 _group(tlist, sql.TypedLiteral, match, valid_prev, valid_next, 

135 post, extend=False) 

136 _group(tlist, sql.TypedLiteral, match_to_extend, valid_prev, valid_final, 

137 post, extend=True) 

138 

139 

140def group_period(tlist): 

141 def match(token): 

142 return token.match(T.Punctuation, '.') 

143 

144 def valid_prev(token): 

145 sqlcls = sql.SquareBrackets, sql.Identifier 

146 ttypes = T.Name, T.String.Symbol 

147 return imt(token, i=sqlcls, t=ttypes) 

148 

149 def valid_next(token): 

150 # issue261, allow invalid next token 

151 return True 

152 

153 def post(tlist, pidx, tidx, nidx): 

154 # next_ validation is being performed here. issue261 

155 sqlcls = sql.SquareBrackets, sql.Function 

156 ttypes = T.Name, T.String.Symbol, T.Wildcard 

157 next_ = tlist[nidx] if nidx is not None else None 

158 valid_next = imt(next_, i=sqlcls, t=ttypes) 

159 

160 return (pidx, nidx) if valid_next else (pidx, tidx) 

161 

162 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

163 

164 

165def group_as(tlist): 

166 def match(token): 

167 return token.is_keyword and token.normalized == 'AS' 

168 

169 def valid_prev(token): 

170 return token.normalized == 'NULL' or not token.is_keyword 

171 

172 def valid_next(token): 

173 ttypes = T.DML, T.DDL, T.CTE 

174 return not imt(token, t=ttypes) and token is not None 

175 

176 def post(tlist, pidx, tidx, nidx): 

177 return pidx, nidx 

178 

179 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

180 

181 

182def group_assignment(tlist): 

183 def match(token): 

184 return token.match(T.Assignment, ':=') 

185 

186 def valid(token): 

187 return token is not None and token.ttype not in (T.Keyword) 

188 

189 def post(tlist, pidx, tidx, nidx): 

190 m_semicolon = T.Punctuation, ';' 

191 snidx, _ = tlist.token_next_by(m=m_semicolon, idx=nidx) 

192 nidx = snidx or nidx 

193 return pidx, nidx 

194 

195 valid_prev = valid_next = valid 

196 _group(tlist, sql.Assignment, match, valid_prev, valid_next, post) 

197 

198 

199def group_comparison(tlist): 

200 sqlcls = (sql.Parenthesis, sql.Function, sql.Identifier, 

201 sql.Operation, sql.TypedLiteral) 

202 ttypes = T_NUMERICAL + T_STRING + T_NAME 

203 

204 def match(token): 

205 return token.ttype == T.Operator.Comparison 

206 

207 def valid(token): 

208 if imt(token, t=ttypes, i=sqlcls): 

209 return True 

210 elif token and token.is_keyword and token.normalized == 'NULL': 

211 return True 

212 else: 

213 return False 

214 

215 def post(tlist, pidx, tidx, nidx): 

216 return pidx, nidx 

217 

218 valid_prev = valid_next = valid 

219 _group(tlist, sql.Comparison, match, 

220 valid_prev, valid_next, post, extend=False) 

221 

222 

223@recurse(sql.Identifier) 

224def group_identifier(tlist): 

225 ttypes = (T.String.Symbol, T.Name) 

226 

227 tidx, token = tlist.token_next_by(t=ttypes) 

228 while token: 

229 tlist.group_tokens(sql.Identifier, tidx, tidx) 

230 tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) 

231 

232 

233def group_arrays(tlist): 

234 sqlcls = sql.SquareBrackets, sql.Identifier, sql.Function 

235 ttypes = T.Name, T.String.Symbol 

236 

237 def match(token): 

238 return isinstance(token, sql.SquareBrackets) 

239 

240 def valid_prev(token): 

241 return imt(token, i=sqlcls, t=ttypes) 

242 

243 def valid_next(token): 

244 return True 

245 

246 def post(tlist, pidx, tidx, nidx): 

247 return pidx, tidx 

248 

249 _group(tlist, sql.Identifier, match, 

250 valid_prev, valid_next, post, extend=True, recurse=False) 

251 

252 

253def group_operator(tlist): 

254 ttypes = T_NUMERICAL + T_STRING + T_NAME 

255 sqlcls = (sql.SquareBrackets, sql.Parenthesis, sql.Function, 

256 sql.Identifier, sql.Operation, sql.TypedLiteral) 

257 

258 def match(token): 

259 return imt(token, t=(T.Operator, T.Wildcard)) 

260 

261 def valid(token): 

262 return imt(token, i=sqlcls, t=ttypes) \ 

263 or (token and token.match( 

264 T.Keyword, 

265 ('CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP'))) 

266 

267 def post(tlist, pidx, tidx, nidx): 

268 tlist[tidx].ttype = T.Operator 

269 return pidx, nidx 

270 

271 valid_prev = valid_next = valid 

272 _group(tlist, sql.Operation, match, 

273 valid_prev, valid_next, post, extend=False) 

274 

275 

276def group_identifier_list(tlist): 

277 m_role = T.Keyword, ('null', 'role') 

278 sqlcls = (sql.Function, sql.Case, sql.Identifier, sql.Comparison, 

279 sql.IdentifierList, sql.Operation) 

280 ttypes = (T_NUMERICAL + T_STRING + T_NAME 

281 + (T.Keyword, T.Comment, T.Wildcard)) 

282 

283 def match(token): 

284 return token.match(T.Punctuation, ',') 

285 

286 def valid(token): 

287 return imt(token, i=sqlcls, m=m_role, t=ttypes) 

288 

289 def post(tlist, pidx, tidx, nidx): 

290 return pidx, nidx 

291 

292 valid_prev = valid_next = valid 

293 _group(tlist, sql.IdentifierList, match, 

294 valid_prev, valid_next, post, extend=True) 

295 

296 

297@recurse(sql.Comment) 

298def group_comments(tlist): 

299 tidx, token = tlist.token_next_by(t=T.Comment) 

300 while token: 

301 eidx, end = tlist.token_not_matching( 

302 lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace, idx=tidx) 

303 if end is not None: 

304 eidx, end = tlist.token_prev(eidx, skip_ws=False) 

305 tlist.group_tokens(sql.Comment, tidx, eidx) 

306 

307 tidx, token = tlist.token_next_by(t=T.Comment, idx=tidx) 

308 

309 

310@recurse(sql.Where) 

311def group_where(tlist): 

312 tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN) 

313 while token: 

314 eidx, end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=tidx) 

315 

316 if end is None: 

317 end = tlist._groupable_tokens[-1] 

318 else: 

319 end = tlist.tokens[eidx - 1] 

320 # TODO: convert this to eidx instead of end token. 

321 # i think above values are len(tlist) and eidx-1 

322 eidx = tlist.token_index(end) 

323 tlist.group_tokens(sql.Where, tidx, eidx) 

324 tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=tidx) 

325 

326 

327@recurse() 

328def group_aliased(tlist): 

329 I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier, 

330 sql.Operation, sql.Comparison) 

331 

332 tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number) 

333 while token: 

334 nidx, next_ = tlist.token_next(tidx) 

335 if isinstance(next_, sql.Identifier): 

336 tlist.group_tokens(sql.Identifier, tidx, nidx, extend=True) 

337 tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=tidx) 

338 

339 

340@recurse(sql.Function) 

341def group_functions(tlist): 

342 has_create = False 

343 has_table = False 

344 has_as = False 

345 for tmp_token in tlist.tokens: 

346 if tmp_token.value.upper() == 'CREATE': 

347 has_create = True 

348 if tmp_token.value.upper() == 'TABLE': 

349 has_table = True 

350 if tmp_token.value == 'AS': 

351 has_as = True 

352 if has_create and has_table and not has_as: 

353 return 

354 

355 tidx, token = tlist.token_next_by(t=T.Name) 

356 while token: 

357 nidx, next_ = tlist.token_next(tidx) 

358 if isinstance(next_, sql.Parenthesis): 

359 tlist.group_tokens(sql.Function, tidx, nidx) 

360 tidx, token = tlist.token_next_by(t=T.Name, idx=tidx) 

361 

362 

363def group_order(tlist): 

364 """Group together Identifier and Asc/Desc token""" 

365 tidx, token = tlist.token_next_by(t=T.Keyword.Order) 

366 while token: 

367 pidx, prev_ = tlist.token_prev(tidx) 

368 if imt(prev_, i=sql.Identifier, t=T.Number): 

369 tlist.group_tokens(sql.Identifier, pidx, tidx) 

370 tidx = pidx 

371 tidx, token = tlist.token_next_by(t=T.Keyword.Order, idx=tidx) 

372 

373 

374@recurse() 

375def align_comments(tlist): 

376 tidx, token = tlist.token_next_by(i=sql.Comment) 

377 while token: 

378 pidx, prev_ = tlist.token_prev(tidx) 

379 if isinstance(prev_, sql.TokenList): 

380 tlist.group_tokens(sql.TokenList, pidx, tidx, extend=True) 

381 tidx = pidx 

382 tidx, token = tlist.token_next_by(i=sql.Comment, idx=tidx) 

383 

384 

385def group_values(tlist): 

386 tidx, token = tlist.token_next_by(m=(T.Keyword, 'VALUES')) 

387 start_idx = tidx 

388 end_idx = -1 

389 while token: 

390 if isinstance(token, sql.Parenthesis): 

391 end_idx = tidx 

392 tidx, token = tlist.token_next(tidx) 

393 if end_idx != -1: 

394 tlist.group_tokens(sql.Values, start_idx, end_idx, extend=True) 

395 

396 

397def group(stmt): 

398 for func in [ 

399 group_comments, 

400 

401 # _group_matching 

402 group_brackets, 

403 group_parenthesis, 

404 group_case, 

405 group_if, 

406 group_for, 

407 group_begin, 

408 

409 group_functions, 

410 group_where, 

411 group_period, 

412 group_arrays, 

413 group_identifier, 

414 group_order, 

415 group_typecasts, 

416 group_tzcasts, 

417 group_typed_literal, 

418 group_operator, 

419 group_comparison, 

420 group_as, 

421 group_aliased, 

422 group_assignment, 

423 

424 align_comments, 

425 group_identifier_list, 

426 group_values, 

427 ]: 

428 func(stmt) 

429 return stmt 

430 

431 

432def _group(tlist, cls, match, 432 ↛ exitline 432 didn't jump to the function exit

433 valid_prev=lambda t: True, 

434 valid_next=lambda t: True, 

435 post=None, 

436 extend=True, 

437 recurse=True 

438 ): 

439 """Groups together tokens that are joined by a middle token. i.e. x < y""" 

440 

441 tidx_offset = 0 

442 pidx, prev_ = None, None 

443 for idx, token in enumerate(list(tlist)): 

444 tidx = idx - tidx_offset 

445 if tidx < 0: # tidx shouldn't get negative 

446 continue 

447 

448 if token.is_whitespace: 

449 continue 

450 

451 if recurse and token.is_group and not isinstance(token, cls): 

452 _group(token, cls, match, valid_prev, valid_next, post, extend) 

453 

454 if match(token): 

455 nidx, next_ = tlist.token_next(tidx) 

456 if prev_ and valid_prev(prev_) and valid_next(next_): 

457 from_idx, to_idx = post(tlist, pidx, tidx, nidx) 

458 grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend) 

459 

460 tidx_offset += to_idx - from_idx 

461 pidx, prev_ = from_idx, grp 

462 continue 

463 

464 pidx, prev_ = tidx, token