tokens.py 1.6 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768
  1. #
  2. # Copyright (C) 2009-2020 the sqlparse authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of python-sqlparse and is released under
  6. # the BSD License: https://opensource.org/licenses/BSD-3-Clause
  7. #
  8. # The Token implementation is based on pygment's token system written
  9. # by Georg Brandl.
  10. # http://pygments.org/
  11. """Tokens"""
  12. class _TokenType(tuple):
  13. parent = None
  14. def __contains__(self, item):
  15. return item is not None and (self is item or item[:len(self)] == self)
  16. def __getattr__(self, name):
  17. new = _TokenType(self + (name,))
  18. setattr(self, name, new)
  19. new.parent = self
  20. return new
  21. def __repr__(self):
  22. # self can be False only if its the `root` i.e. Token itself
  23. return 'Token' + ('.' if self else '') + '.'.join(self)
  24. Token = _TokenType()
  25. # Special token types
  26. Text = Token.Text
  27. Whitespace = Text.Whitespace
  28. Newline = Whitespace.Newline
  29. Error = Token.Error
  30. # Text that doesn't belong to this lexer (e.g. HTML in PHP)
  31. Other = Token.Other
  32. # Common token types for source code
  33. Keyword = Token.Keyword
  34. Name = Token.Name
  35. Literal = Token.Literal
  36. String = Literal.String
  37. Number = Literal.Number
  38. Punctuation = Token.Punctuation
  39. Operator = Token.Operator
  40. Comparison = Operator.Comparison
  41. Wildcard = Token.Wildcard
  42. Comment = Token.Comment
  43. Assignment = Token.Assignment
  44. # Generic types for non-source code
  45. Generic = Token.Generic
  46. Command = Generic.Command
  47. # String and some others are not direct children of Token.
  48. # alias them:
  49. Token.Token = Token
  50. Token.String = String
  51. Token.Number = Number
  52. # SQL specific tokens
  53. DML = Keyword.DML
  54. DDL = Keyword.DDL
  55. CTE = Keyword.CTE