eric: comparison eric6/ThirdParty/Pygments/pygments/lexers/sql.py

-:a3cf077a8db3
+:25f42e208e08
 - handles psql backslash commands.
 The ``tests/examplefiles`` contains a few test files with data to be
 parsed by these lexers.
-:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+:copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
 :license: BSD, see LICENSE for details.
 """
 import re
 from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words
 from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \
-Keyword, Name, String, Number, Generic
+Keyword, Name, String, Number, Generic, Literal
 from pygments.lexers import get_lexer_by_name, ClassNotFound
 from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \
 PSEUDO_TYPES, PLPGSQL_KEYWORDS
+from pygments.lexers._mysql_builtins import \
+MYSQL_CONSTANTS, \
+MYSQL_DATATYPES, \
+MYSQL_FUNCTIONS, \
+MYSQL_KEYWORDS, \
+MYSQL_OPTIMIZER_HINTS
 from pygments.lexers import _tsql_builtins
 __all__ = ['PostgresLexer', 'PlPgsqlLexer', 'PostgresConsoleLexer',
 'SqlLexer', 'TransactSqlLexer', 'MySqlLexer',
 seem to suggest that regexp lexers are not really subclassable.
 """
 def get_tokens_unprocessed(self, text, *args):
 # Have a copy of the entire text to be used by `language_callback`.
 self.text = text
-for x in super(PostgresBase, self).get_tokens_unprocessed(
+yield from super().get_tokens_unprocessed(text, *args)
-text, *args):
-yield x
 def _get_lexer(self, lang):
 if lang.lower() == 'sql':
 return get_lexer_by_name('postgresql', **self.options)
 insertions = []
 for line in lines:
 # Identify a shell prompt in case of psql commandline example
 if line.startswith('$') and not curcode:
 lexer = get_lexer_by_name('console', **self.options)
-for x in lexer.get_tokens_unprocessed(line):
+yield from lexer.get_tokens_unprocessed(line)
-yield x
 break
 # Identify a psql prompt
 mprompt = re_prompt.match(line)
 if mprompt is not None:
 if re_psql_command.match(curcode) \
 or re_end_command.search(curcode):
 break
 # Emit the combined stream of command and prompt(s)
-for item in do_insertions(insertions,
+yield from do_insertions(insertions,
-sql.get_tokens_unprocessed(curcode)):
+sql.get_tokens_unprocessed(curcode))
-yield item
 # Emit the output lines
 out_token = Generic.Output
 for line in lines:
 mprompt = re_prompt.match(line)
 # Use re.UNICODE to allow non ASCII letters in names.
 flags = re.IGNORECASE | re.UNICODE
 tokens = {
 'root': [
 (r'\s+', Whitespace),
-(r'(?m)--.*?$\n?', Comment.Single),
+(r'--.*?$\n?', Comment.Single),
 (r'/\*', Comment.Multiline, 'multiline-comments'),
 (words(_tsql_builtins.OPERATORS), Operator),
 (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word),
 (words(_tsql_builtins.TYPES, suffix=r'\b'), Name.Class),
 (words(_tsql_builtins.FUNCTIONS, suffix=r'\b'), Name.Function),
 rating += 0.1
 return rating
 class MySqlLexer(RegexLexer):
-"""
+"""The Oracle MySQL lexer.
-Special lexer for MySQL.
+This lexer does not attempt to maintain strict compatibility with
+MariaDB syntax or keywords. Although MySQL and MariaDB's common code
+history suggests there may be significant overlap between the two,
+compatibility between the two is not a target for this lexer.
 """
 name = 'MySQL'
 aliases = ['mysql']
 mimetypes = ['text/x-mysql']
 flags = re.IGNORECASE
 tokens = {
 'root': [
 (r'\s+', Text),
-(r'(#|--\s+).*\n?', Comment.Single),
-(r'/\*', Comment.Multiline, 'multiline-comments'),
+# Comments
+(r'(?:#|--\s+).*', Comment.Single),
+(r'/\*\+', Comment.Special, 'optimizer-hints'),
+(r'/\*', Comment.Multiline, 'multiline-comment'),
+# Hexadecimal literals
+(r"x'([0-9a-f]{2})+'", Number.Hex),  # MySQL requires paired hex characters in this form.
+(r'0x[0-9a-f]+', Number.Hex),
+# Binary literals
+(r"b'[01]+'", Number.Bin),
+(r'0b[01]+', Number.Bin),
+# Numeric literals
+(r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float),  # Mandatory integer, optional fraction and exponent
+(r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),  # Mandatory fraction, optional integer and exponent
+(r'[0-9]+e[+-]?[0-9]+', Number.Float),  # Exponents with integer significands are still floats
 (r'[0-9]+', Number.Integer),
-(r'[0-9]*\.[0-9]+(e[+-][0-9]+)', Number.Float),
-(r"'(\\\\|\\'|''|[^'])*'", String.Single),
+# Date literals
-(r'"(\\\\|\\"|""|[^"])*"', String.Double),
+(r"\{\s*d\s*(?P<quote>['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}",
-(r"`(\\\\|\\`|``|[^`])*`", String.Symbol),
+Literal.Date),
-(r'[+*/<>=~!@#%^&|`?-]', Operator),
-(r'\b(tinyint|smallint|mediumint|int|integer|bigint|date|'
+# Time literals
-r'datetime|time|bit|bool|tinytext|mediumtext|longtext|text|'
+(r"\{\s*t\s*(?P<quote>['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}",
-r'tinyblob|mediumblob|longblob|blob|float|double|double\s+'
+Literal.Date),
-r'precision|real|numeric|dec|decimal|timestamp|year|char|'
-r'varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?',
+# Timestamp literals
-bygroups(Keyword.Type, Text, Punctuation)),
+(
-(r'\b(add|all|alter|analyze|and|as|asc|asensitive|before|between|'
+r"\{\s*ts\s*(?P<quote>['\"])\s*"
-r'bigint|binary|blob|both|by|call|cascade|case|change|char|'
+r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}"  # Date part
-r'character|check|collate|column|condition|constraint|continue|'
+r"\s+"  # Whitespace between date and time
-r'convert|create|cross|current_date|current_time|'
+r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?"  # Time part
-r'current_timestamp|current_user|cursor|database|databases|'
+r"\s*(?P=quote)\s*\}",
-r'day_hour|day_microsecond|day_minute|day_second|dec|decimal|'
+Literal.Date
-r'declare|default|delayed|delete|desc|describe|deterministic|'
+),
-r'distinct|distinctrow|div|double|drop|dual|each|else|elseif|'
-r'enclosed|escaped|exists|exit|explain|fetch|flush|float|float4|'
+# String literals
-r'float8|for|force|foreign|from|fulltext|grant|group|having|'
+(r"'", String.Single, 'single-quoted-string'),
-r'high_priority|hour_microsecond|hour_minute|hour_second|if|'
+(r'"', String.Double, 'double-quoted-string'),
-r'ignore|in|index|infile|inner|inout|insensitive|insert|int|'
-r'int1|int2|int3|int4|int8|integer|interval|into|is|iterate|'
+# Variables
-r'join|key|keys|kill|leading|leave|left|like|limit|lines|load|'
+(r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable),
-r'localtime|localtimestamp|lock|long|loop|low_priority|match|'
+(r'@[a-z0-9_$.]+', Name.Variable),
-r'minute_microsecond|minute_second|mod|modifies|natural|'
+(r"@'", Name.Variable, 'single-quoted-variable'),
-r'no_write_to_binlog|not|numeric|on|optimize|option|optionally|'
+(r'@"', Name.Variable, 'double-quoted-variable'),
-r'or|order|out|outer|outfile|precision|primary|procedure|purge|'
+(r"@`", Name.Variable, 'backtick-quoted-variable'),
-r'raid0|read|reads|real|references|regexp|release|rename|repeat|'
+(r'\?', Name.Variable),  # For demonstrating prepared statements
-r'replace|require|restrict|return|revoke|right|rlike|schema|'
-r'schemas|second_microsecond|select|sensitive|separator|set|'
+# Operators
-r'show|smallint|soname|spatial|specific|sql|sql_big_result|'
+(r'[!%&*+/:<=>^|~-]+', Operator),
-r'sql_calc_found_rows|sql_small_result|sqlexception|sqlstate|'
-r'sqlwarning|ssl|starting|straight_join|table|terminated|then|'
+# Exceptions; these words tokenize differently in different contexts.
-r'to|trailing|trigger|undo|union|unique|unlock|unsigned|update|'
+(r'\b(set)(?!\s*\()', Keyword),
-r'usage|use|using|utc_date|utc_time|utc_timestamp|values|'
+(r'\b(character)(\s+)(set)\b', bygroups(Keyword, Text, Keyword)),
-r'varying|when|where|while|with|write|x509|xor|year_month|'
+# In all other known cases, "SET" is tokenized by MYSQL_DATATYPES.
-r'zerofill)\b', Keyword),
-# TODO: this list is not complete
+(words(MYSQL_CONSTANTS, prefix=r'\b', suffix=r'\b'), Name.Constant),
-(r'\b(auto_increment|engine|charset|tables)\b', Keyword.Pseudo),
+(words(MYSQL_DATATYPES, prefix=r'\b', suffix=r'\b'), Keyword.Type),
-(r'(true|false|null)', Name.Constant),
+(words(MYSQL_KEYWORDS, prefix=r'\b', suffix=r'\b'), Keyword),
-(r'([a-z_]\w*)(\s*)(\()',
+(words(MYSQL_FUNCTIONS, prefix=r'\b', suffix=r'\b(\s*)(\()'),
 bygroups(Name.Function, Text, Punctuation)),
-(r'[a-z_]\w*', Name),
-(r'@[a-z0-9]*[._]*[a-z0-9]*', Name.Variable),
+# Schema object names
-(r'[;:()\[\],.]', Punctuation)
+#
-],
+# Note: Although the first regex supports unquoted all-numeric
-'multiline-comments': [
+# identifiers, this will not be a problem in practice because
-(r'/\*', Comment.Multiline, 'multiline-comments'),
+# numeric literals have already been handled above.
+#
+('[0-9a-z$_\u0080-\uffff]+', Name),
+(r'`', Name, 'schema-object-name'),
+# Punctuation
+(r'[(),.;]', Punctuation),
+],
+# Multiline comment substates
+# ---------------------------
+'optimizer-hints': [
+(r'[^*a-z]+', Comment.Special),
+(r'\*/', Comment.Special, '#pop'),
+(words(MYSQL_OPTIMIZER_HINTS, suffix=r'\b'), Comment.Preproc),
+('[a-z]+', Comment.Special),
+(r'\*', Comment.Special),
+],
+'multiline-comment': [
+(r'[^*]+', Comment.Multiline),
 (r'\*/', Comment.Multiline, '#pop'),
-(r'[^/*]+', Comment.Multiline),
+(r'\*', Comment.Multiline),
-(r'[/*]', Comment.Multiline)
+],
-]
+# String substates
+# ----------------
+'single-quoted-string': [
+(r"[^'\\]+", String.Single),
+(r"''", String.Escape),
+(r"""\\[0'"bnrtZ\\%_]""", String.Escape),
+(r"'", String.Single, '#pop'),
+],
+'double-quoted-string': [
+(r'[^"\\]+', String.Double),
+(r'""', String.Escape),
+(r"""\\[0'"bnrtZ\\%_]""", String.Escape),
+(r'"', String.Double, '#pop'),
+],
+# Variable substates
+# ------------------
+'single-quoted-variable': [
+(r"[^']+", Name.Variable),
+(r"''", Name.Variable),
+(r"'", Name.Variable, '#pop'),
+],
+'double-quoted-variable': [
+(r'[^"]+', Name.Variable),
+(r'""', Name.Variable),
+(r'"', Name.Variable, '#pop'),
+],
+'backtick-quoted-variable': [
+(r'[^`]+', Name.Variable),
+(r'``', Name.Variable),
+(r'`', Name.Variable, '#pop'),
+],
+# Schema object name substates
+# ----------------------------
+#
+# Backtick-quoted schema object names support escape characters.
+# It may be desirable to tokenize escape sequences differently,
+# but currently Pygments does not have an obvious token type for
+# this unique situation (for example, "Name.Escape").
+#
+'schema-object-name': [
+(r'[^`\\]+', Name),
+(r'(?:\\\\|\\`|``)', Name),  # This could be an escaped name token type.
+(r'`', Name, '#pop'),
+],
 }
 def analyse_text(text):
 rating = 0
 name_between_backtick_count = len(
 insertions.append((len(curcode),
 [(0, Generic.Prompt, line[:8])]))
 curcode += line[8:]
 else:
 if curcode:
-for item in do_insertions(insertions,
+yield from do_insertions(insertions,
-sql.get_tokens_unprocessed(curcode)):
+sql.get_tokens_unprocessed(curcode))
-yield item
 curcode = ''
 insertions = []
 if line.startswith('SQL error: '):
 yield (match.start(), Generic.Traceback, line)
 else:
 yield (match.start(), Generic.Output, line)
 if curcode:
-for item in do_insertions(insertions,
+yield from do_insertions(insertions,
-sql.get_tokens_unprocessed(curcode)):
+sql.get_tokens_unprocessed(curcode))
-yield item
 class RqlLexer(RegexLexer):
 """
 Lexer for Relation Query Language.

Mercurial Repositories > eric / file comparison

comparison: eric6/ThirdParty/Pygments/pygments/lexers/sql.py

eric6/ThirdParty/Pygments/pygments/lexers/sql.py