173 yield offset, Text, content |
173 yield offset, Text, content |
174 |
174 |
175 tokens = { |
175 tokens = { |
176 'root': [ |
176 'root': [ |
177 (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH)( +)([^ ]+)( +)' |
177 (r'(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH)( +)([^ ]+)( +)' |
178 r'(HTTP)(/)(1\.[01]|2|3)(\r?\n|\Z)', |
178 r'(HTTP)(/)(1\.[01]|2(?:\.0)?|3)(\r?\n|\Z)', |
179 bygroups(Name.Function, Text, Name.Namespace, Text, |
179 bygroups(Name.Function, Text, Name.Namespace, Text, |
180 Keyword.Reserved, Operator, Number, Text), |
180 Keyword.Reserved, Operator, Number, Text), |
181 'headers'), |
181 'headers'), |
182 (r'(HTTP)(/)(1\.[01]|2|3)( +)(\d{3})(?:( +)([^\r\n]+))?(\r?\n|\Z)', |
182 (r'(HTTP)(/)(1\.[01]|2(?:\.0)?|3)( +)(\d{3})(?:( +)([^\r\n]*))?(\r?\n|\Z)', |
183 bygroups(Keyword.Reserved, Operator, Number, Text, Number, Text, |
183 bygroups(Keyword.Reserved, Operator, Number, Text, Number, Text, |
184 Name.Exception, Text), |
184 Name.Exception, Text), |
185 'headers'), |
185 'headers'), |
186 ], |
186 ], |
187 'headers': [ |
187 'headers': [ |
324 else: |
324 else: |
325 lexer = guess_lexer(code.strip()) |
325 lexer = guess_lexer(code.strip()) |
326 except ClassNotFound: |
326 except ClassNotFound: |
327 lexer = get_lexer_by_name('text') |
327 lexer = get_lexer_by_name('text') |
328 |
328 |
329 for item in lexer.get_tokens_unprocessed(code): |
329 yield from lexer.get_tokens_unprocessed(code) |
330 yield item |
330 |
331 |
331 tokens = { |
332 tokens = { |
332 'root': [ |
333 'root': [ |
333 (r'\fmessage\{\s*', Keyword, ('message', 'message-attr')), |
334 (r'\fmessage{\s*', Keyword, ('message', 'message-attr')), |
|
335 ], |
334 ], |
336 'message-attr': [ |
335 'message-attr': [ |
337 (r'(\s*id:\s*)([^\s]+)', bygroups(Name.Attribute, String)), |
336 (r'(\s*id:\s*)(\S+)', bygroups(Name.Attribute, String)), |
338 (r'(\s*(?:depth|match|excluded):\s*)(\d+)', |
337 (r'(\s*(?:depth|match|excluded):\s*)(\d+)', |
339 bygroups(Name.Attribute, Number.Integer)), |
338 bygroups(Name.Attribute, Number.Integer)), |
340 (r'(\s*filename:\s*)(.+\n)', |
339 (r'(\s*filename:\s*)(.+\n)', |
341 bygroups(Name.Attribute, String)), |
340 bygroups(Name.Attribute, String)), |
342 default('#pop'), |
341 default('#pop'), |
343 ], |
342 ], |
344 'message': [ |
343 'message': [ |
345 (r'\fmessage}\n', Keyword, '#pop'), |
344 (r'\fmessage\}\n', Keyword, '#pop'), |
346 (r'\fheader{\n', Keyword, 'header'), |
345 (r'\fheader\{\n', Keyword, 'header'), |
347 (r'\fbody{\n', Keyword, 'body'), |
346 (r'\fbody\{\n', Keyword, 'body'), |
348 ], |
347 ], |
349 'header': [ |
348 'header': [ |
350 (r'\fheader}\n', Keyword, '#pop'), |
349 (r'\fheader\}\n', Keyword, '#pop'), |
351 (r'((?:Subject|From|To|Cc|Date):\s*)(.*\n)', |
350 (r'((?:Subject|From|To|Cc|Date):\s*)(.*\n)', |
352 bygroups(Name.Attribute, String)), |
351 bygroups(Name.Attribute, String)), |
353 (r'(.*)(\s*\(.*\))(\s*\(.*\)\n)', |
352 (r'(.*)(\s*\(.*\))(\s*\(.*\)\n)', |
354 bygroups(Generic.Strong, Literal, Name.Tag)), |
353 bygroups(Generic.Strong, Literal, Name.Tag)), |
355 ], |
354 ], |
356 'body': [ |
355 'body': [ |
357 (r'\fpart{\n', Keyword, 'part'), |
356 (r'\fpart\{\n', Keyword, 'part'), |
358 (r'\f(part|attachment){\s*', Keyword, ('part', 'part-attr')), |
357 (r'\f(part|attachment)\{\s*', Keyword, ('part', 'part-attr')), |
359 (r'\fbody}\n', Keyword, '#pop'), |
358 (r'\fbody\}\n', Keyword, '#pop'), |
360 ], |
359 ], |
361 'part-attr': [ |
360 'part-attr': [ |
362 (r'(ID:\s*)(\d+)', bygroups(Name.Attribute, Number.Integer)), |
361 (r'(ID:\s*)(\d+)', bygroups(Name.Attribute, Number.Integer)), |
363 (r'(,\s*)((?:Filename|Content-id):\s*)([^,]+)', |
362 (r'(,\s*)((?:Filename|Content-id):\s*)([^,]+)', |
364 bygroups(Punctuation, Name.Attribute, String)), |
363 bygroups(Punctuation, Name.Attribute, String)), |
365 (r'(,\s*)(Content-type:\s*)(.+\n)', |
364 (r'(,\s*)(Content-type:\s*)(.+\n)', |
366 bygroups(Punctuation, Name.Attribute, String)), |
365 bygroups(Punctuation, Name.Attribute, String)), |
367 default('#pop'), |
366 default('#pop'), |
368 ], |
367 ], |
369 'part': [ |
368 'part': [ |
370 (r'\f(?:part|attachment)}\n', Keyword, '#pop'), |
369 (r'\f(?:part|attachment)\}\n', Keyword, '#pop'), |
371 (r'\f(?:part|attachment){\s*', Keyword, ('#push', 'part-attr')), |
370 (r'\f(?:part|attachment)\{\s*', Keyword, ('#push', 'part-attr')), |
372 (r'^Non-text part: .*\n', Comment), |
371 (r'^Non-text part: .*\n', Comment), |
373 (r'(?s)(.*?(?=\f(?:part|attachment)}\n))', _highlight_code), |
372 (r'(?s)(.*?(?=\f(?:part|attachment)\}\n))', _highlight_code), |
374 ], |
373 ], |
375 } |
374 } |
376 |
375 |
377 def analyse_text(text): |
376 def analyse_text(text): |
378 return 1.0 if text.startswith('\fmessage{') else 0.0 |
377 return 1.0 if text.startswith('\fmessage{') else 0.0 |